From fadad7f762ecf968eeddd9d978edaa5dae662cae Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Thu, 24 Nov 2022 21:28:33 +0200 Subject: [PATCH 01/34] updated IntegrationLauncherConfig.yaml and added to this class suportDBT and normalizationImage fields. Added to the GenerateInputActivityImpl and TemporalClient classes code parts for read destination_definition.yaml and get suportDBT and normalizationImage fields. Added logging and comparing normalization images from NormalizationRunnerFactory and destination_definition.yaml --- airbyte-commons-temporal/build.gradle | 1 + .../commons/temporal/TemporalClient.java | 35 +++++++++++++++---- .../commons/temporal/TemporalClientTest.java | 4 +-- .../DefaultNormalizationRunner.java | 15 ++++++++ .../NormalizationRunnerFactory.java | 17 +++++++-- .../NormalizationRunnerFactoryTest.java | 5 +-- .../DbtJobOrchestrator.java | 3 +- .../NormalizationJobOrchestrator.java | 3 +- .../DestinationAcceptanceTest.java | 8 ++--- .../IntegrationLauncherConfig.yaml | 5 +++ .../activities/GenerateInputActivityImpl.java | 17 ++++++++- .../sync/DbtTransformationActivityImpl.java | 3 +- .../sync/NormalizationActivityImpl.java | 7 +++- 13 files changed, 102 insertions(+), 21 deletions(-) diff --git a/airbyte-commons-temporal/build.gradle b/airbyte-commons-temporal/build.gradle index 7e553f1c7e94..347bddcc1db3 100644 --- a/airbyte-commons-temporal/build.gradle +++ b/airbyte-commons-temporal/build.gradle @@ -15,6 +15,7 @@ dependencies { implementation project(':airbyte-config:config-models') implementation project(':airbyte-config:config-persistence') + implementation project(':airbyte-config:init') implementation project(':airbyte-metrics:metrics-lib') implementation project(':airbyte-persistence:job-persistence') implementation project(':airbyte-protocol:protocol-models') diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java index 9abceab23c75..fb80c5973bd3 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java @@ -23,9 +23,11 @@ import io.airbyte.config.JobGetSpecConfig; import io.airbyte.config.JobSyncConfig; import io.airbyte.config.StandardCheckConnectionInput; +import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardDiscoverCatalogInput; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; +import io.airbyte.config.init.LocalDefinitionsProvider; import io.airbyte.config.persistence.StreamResetPersistence; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; @@ -378,11 +380,32 @@ public TemporalResponse submitSync(final long jobId, final i .withDockerImage(config.getSourceDockerImage()) .withProtocolVersion(config.getSourceProtocolVersion()); - final IntegrationLauncherConfig destinationLauncherConfig = new IntegrationLauncherConfig() - .withJobId(String.valueOf(jobId)) - .withAttemptId((long) attempt) - .withDockerImage(config.getDestinationDockerImage()) - .withProtocolVersion(config.getDestinationProtocolVersion()); + IntegrationLauncherConfig destinationLauncherConfig; + try { + LocalDefinitionsProvider provider = new LocalDefinitionsProvider(LocalDefinitionsProvider.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS); + List destinationDefinitionList = provider.getDestinationDefinitions(); + Optional optionalDestinationDefinition = destinationDefinitionList.stream() + .filter(destinationDefinition -> config.getDestinationDockerImage() + .equalsIgnoreCase(destinationDefinition.getDockerRepository() + ":" + destinationDefinition.getDockerImageTag())) + .findFirst(); + final String destinationNormalizationDockerImage = optionalDestinationDefinition.map(standardDestinationDefinition -> String.format("%s:%s", + standardDestinationDefinition.getNormalizationRepository(), standardDestinationDefinition.getNormalizationTag())).orElse(null); + final boolean supportDBT = optionalDestinationDefinition.isPresent() ? optionalDestinationDefinition.get().getSupportsDbt() : false; + destinationLauncherConfig = new IntegrationLauncherConfig() + .withJobId(String.valueOf(jobId)) + .withAttemptId((long) attempt) + .withDockerImage(config.getDestinationDockerImage()) + .withProtocolVersion(config.getDestinationProtocolVersion()) + .withDockerNormalizationImage(destinationNormalizationDockerImage) + .withSupportDBT(supportDBT); + } catch (IOException ignored) { + destinationLauncherConfig = new IntegrationLauncherConfig() + .withJobId(String.valueOf(jobId)) + .withAttemptId((long) attempt) + .withDockerImage(config.getDestinationDockerImage()) + .withProtocolVersion(config.getDestinationProtocolVersion()); + } + final IntegrationLauncherConfig finalDestinationLauncherConfig = destinationLauncherConfig; final StandardSyncInput input = new StandardSyncInput() .withNamespaceDefinition(config.getNamespaceDefinition()) @@ -401,7 +424,7 @@ public TemporalResponse submitSync(final long jobId, final i () -> getWorkflowStub(SyncWorkflow.class, TemporalJobType.SYNC).run( jobRunConfig, sourceLauncherConfig, - destinationLauncherConfig, + finalDestinationLauncherConfig, input, connectionId)); } diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java index 08e7fd986552..36b3eda38806 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java @@ -270,7 +270,7 @@ void testSubmitSync() { .thenReturn(discoverCatalogWorkflow); final JobSyncConfig syncConfig = new JobSyncConfig() .withSourceDockerImage(IMAGE_NAME1) - .withSourceDockerImage(IMAGE_NAME2) + .withDestinationDockerImage(IMAGE_NAME2) .withSourceConfiguration(Jsons.emptyObject()) .withDestinationConfiguration(Jsons.emptyObject()) .withOperationSequence(List.of()) @@ -342,7 +342,7 @@ void testforceCancelConnection() { final JobSyncConfig syncConfig = new JobSyncConfig() .withSourceDockerImage(IMAGE_NAME1) - .withSourceDockerImage(IMAGE_NAME2) + .withDestinationDockerImage(IMAGE_NAME2) .withSourceConfiguration(Jsons.emptyObject()) .withDestinationConfiguration(Jsons.emptyObject()) .withOperationSequence(List.of()) diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java index 3db73387fec0..ee65ace552d8 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java @@ -31,6 +31,7 @@ import java.io.InputStream; import java.nio.file.Path; import java.util.Collections; +import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -75,6 +76,20 @@ public DefaultNormalizationRunner(final DestinationType destinationType, this.normalizationImageName = normalizationImageName; } + public DefaultNormalizationRunner(final ProcessFactory processFactory, + final String normalizationImage) { + this.processFactory = processFactory; + this.destinationType = getDestinationTypeByNormalizationImageNamePart(normalizationImage); + this.normalizationImageName = normalizationImage; + } + + private static DestinationType getDestinationTypeByNormalizationImageNamePart(final String normalizationImageName) { + return EnumSet.allOf(DestinationType.class).stream() + .filter(destinationType -> normalizationImageName.contains(destinationType.name().toLowerCase())) + .findFirst() + .orElseGet(null); + } + @Override public boolean configureDbt(final String jobId, final int attempt, diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java index d3943e0f7811..abf22541318d 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java @@ -8,8 +8,11 @@ import io.airbyte.workers.normalization.DefaultNormalizationRunner.DestinationType; import io.airbyte.workers.process.ProcessFactory; import java.util.Map; +import java.util.Objects; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.ImmutablePair; +@Slf4j public class NormalizationRunnerFactory { public static final String BASE_NORMALIZATION_IMAGE_NAME = "airbyte/normalization"; @@ -38,12 +41,22 @@ public class NormalizationRunnerFactory { public static NormalizationRunner create(final String connectorImageName, final ProcessFactory processFactory, - final String normalizationVersion) { + final String normalizationVersion, + final String normalizationImage) { final var valuePair = getNormalizationInfoForConnector(connectorImageName); + final String factoryNormalizationImage = String.format("%s:%s", valuePair.getLeft(), normalizationVersion); + log.error("destination normalization runner factory -> {}", factoryNormalizationImage); + if (Objects.nonNull(normalizationImage) + && !normalizationImage.equalsIgnoreCase(factoryNormalizationImage)) { + log.error( + "The normalization image name or tag in the definition file is different from the normalization image or tag in the NormalizationRunnerFactory!"); + log.error( + "the definition file value - {}, the NormalizationRunnerFactory value - {}", normalizationImage, factoryNormalizationImage); + } return new DefaultNormalizationRunner( valuePair.getRight(), processFactory, - String.format("%s:%s", valuePair.getLeft(), normalizationVersion)); + factoryNormalizationImage); } public static ImmutablePair getNormalizationInfoForConnector(final String connectorImageName) { diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/NormalizationRunnerFactoryTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/NormalizationRunnerFactoryTest.java index 4db4152b7373..bc1420c6d1a5 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/NormalizationRunnerFactoryTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/NormalizationRunnerFactoryTest.java @@ -30,11 +30,12 @@ void testMappings() { for (final Entry> entry : NormalizationRunnerFactory.NORMALIZATION_MAPPING.entrySet()) { assertEquals(entry.getValue().getValue(), ((DefaultNormalizationRunner) NormalizationRunnerFactory.create( - String.format("%s:0.1.0", entry.getKey()), processFactory, NORMALIZATION_VERSION)).getDestinationType()); + String.format("%s:0.1.0", entry.getKey()), processFactory, NORMALIZATION_VERSION, String.format("%s:0.1.0", entry.getKey()))) + .getDestinationType()); } assertThrows(IllegalStateException.class, () -> NormalizationRunnerFactory.create("airbyte/destination-csv:0.1.0", processFactory, - NORMALIZATION_VERSION)); + NORMALIZATION_VERSION, "airbyte/destination-csv:0.1.0")); } } diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/DbtJobOrchestrator.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/DbtJobOrchestrator.java index fe3d6da7dab7..34782b91dff3 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/DbtJobOrchestrator.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/DbtJobOrchestrator.java @@ -72,7 +72,8 @@ public Optional runJob() throws Exception { processFactory, NormalizationRunnerFactory.create( destinationLauncherConfig.getDockerImage(), processFactory, - NormalizationRunnerFactory.NORMALIZATION_VERSION))); + NormalizationRunnerFactory.NORMALIZATION_VERSION, + destinationLauncherConfig.getDockerNormalizationImage()))); log.info("Running dbt worker..."); final Path jobRoot = TemporalUtils.getJobRoot(configs.getWorkspaceRoot(), jobRunConfig.getJobId(), jobRunConfig.getAttemptId()); diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/NormalizationJobOrchestrator.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/NormalizationJobOrchestrator.java index 1c602bb35c1b..6cdd3a112aa4 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/NormalizationJobOrchestrator.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/NormalizationJobOrchestrator.java @@ -69,7 +69,8 @@ public Optional runJob() throws Exception { NormalizationRunnerFactory.create( destinationLauncherConfig.getDockerImage(), processFactory, - NormalizationRunnerFactory.NORMALIZATION_VERSION), + NormalizationRunnerFactory.NORMALIZATION_VERSION, + destinationLauncherConfig.getDockerNormalizationImage()), configs.getWorkerEnvironment()); log.info("Running normalization worker..."); diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java index 899f7cb3ff00..4972f6e9d96a 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -533,7 +533,7 @@ public void specNormalizationValueShouldBeCorrect() throws Exception { if (normalizationFromSpec) { boolean normalizationRunnerFactorySupportsDestinationImage; try { - NormalizationRunnerFactory.create(getImageName(), processFactory, NORMALIZATION_VERSION); + NormalizationRunnerFactory.create(getImageName(), processFactory, NORMALIZATION_VERSION, ""); normalizationRunnerFactorySupportsDestinationImage = true; } catch (final IllegalStateException e) { normalizationRunnerFactorySupportsDestinationImage = false; @@ -839,7 +839,7 @@ public void testCustomDbtTransformations() throws Exception { NormalizationRunnerFactory.create( getImageName(), processFactory, - NORMALIZATION_VERSION)); + NORMALIZATION_VERSION, "")); runner.start(); final Path transformationRoot = Files.createDirectories(jobRoot.resolve("transform")); final OperatorDbt dbtConfig = new OperatorDbt() @@ -917,7 +917,7 @@ void testCustomDbtTransformationsFailure() throws Exception { NormalizationRunnerFactory.create( getImageName(), processFactory, - NORMALIZATION_VERSION)); + NORMALIZATION_VERSION, "")); runner.start(); final Path transformationRoot = Files.createDirectories(jobRoot.resolve("transform")); final OperatorDbt dbtConfig = new OperatorDbt() @@ -1283,7 +1283,7 @@ private List runSync( final NormalizationRunner runner = NormalizationRunnerFactory.create( getImageName(), processFactory, - NORMALIZATION_VERSION); + NORMALIZATION_VERSION, ""); runner.start(); final Path normalizationRoot = Files.createDirectories(jobRoot.resolve("normalize")); if (!runner.normalize(JOB_ID, JOB_ATTEMPT, normalizationRoot, diff --git a/airbyte-worker-models/src/main/resources/workers_models/IntegrationLauncherConfig.yaml b/airbyte-worker-models/src/main/resources/workers_models/IntegrationLauncherConfig.yaml index a4c7b6fa63c6..dcdbb219593a 100644 --- a/airbyte-worker-models/src/main/resources/workers_models/IntegrationLauncherConfig.yaml +++ b/airbyte-worker-models/src/main/resources/workers_models/IntegrationLauncherConfig.yaml @@ -16,6 +16,11 @@ properties: type: integer dockerImage: type: string + dockerNormalizationImage: + type: string + supportDBT: + type: boolean + default: false protocolVersion: type: object existingJavaType: io.airbyte.commons.version.Version diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java index de7c26da10f5..b31ad4a07b18 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java @@ -17,7 +17,9 @@ import io.airbyte.config.JobResetConnectionConfig; import io.airbyte.config.JobSyncConfig; import io.airbyte.config.ResetSourceConfiguration; +import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSyncInput; +import io.airbyte.config.init.LocalDefinitionsProvider; import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.persistence.job.JobPersistence; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; @@ -28,6 +30,7 @@ import jakarta.inject.Singleton; import java.util.List; import java.util.Map; +import java.util.Optional; @Singleton @Requires(env = WorkerMode.CONTROL_PLANE) @@ -80,6 +83,16 @@ public GeneratedJobInput getSyncWorkflowInput(final SyncInput input) { final JobRunConfig jobRunConfig = TemporalWorkflowUtils.createJobRunConfig(jobId, attempt); + LocalDefinitionsProvider provider = new LocalDefinitionsProvider(LocalDefinitionsProvider.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS); + List destinationDefinitionList = provider.getDestinationDefinitions(); + Optional optionalDestinationDefinition = destinationDefinitionList.stream() + .filter(destinationDefinition -> config.getDestinationDockerImage() + .equalsIgnoreCase(destinationDefinition.getDockerRepository() + ":" + destinationDefinition.getDockerImageTag())) + .findFirst(); + final String destinationNormalizationDockerImage = optionalDestinationDefinition.map(standardDestinationDefinition -> String.format("%s:%s", + standardDestinationDefinition.getNormalizationRepository(), standardDestinationDefinition.getNormalizationTag())).orElse(null); + final boolean supportDBT = optionalDestinationDefinition.isPresent() ? optionalDestinationDefinition.get().getSupportsDbt() : false; + final IntegrationLauncherConfig sourceLauncherConfig = new IntegrationLauncherConfig() .withJobId(String.valueOf(jobId)) .withAttemptId((long) attempt) @@ -90,7 +103,9 @@ public GeneratedJobInput getSyncWorkflowInput(final SyncInput input) { .withJobId(String.valueOf(jobId)) .withAttemptId((long) attempt) .withDockerImage(config.getDestinationDockerImage()) - .withProtocolVersion(config.getDestinationProtocolVersion()); + .withProtocolVersion(config.getDestinationProtocolVersion()) + .withDockerNormalizationImage(destinationNormalizationDockerImage) + .withSupportDBT(supportDBT); final StandardSyncInput syncInput = new StandardSyncInput() .withNamespaceDefinition(config.getNamespaceDefinition()) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java index 768f397ed86f..ec0ee6f52275 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java @@ -146,7 +146,8 @@ private CheckedSupplier, Exception> getLegacyWork processFactory, NormalizationRunnerFactory.create( destinationLauncherConfig.getDockerImage(), processFactory, - NormalizationRunnerFactory.NORMALIZATION_VERSION))); + NormalizationRunnerFactory.NORMALIZATION_VERSION, + destinationLauncherConfig.getDockerNormalizationImage()))); } private CheckedSupplier, Exception> getContainerLauncherWorkerFactory( diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java index b37e41e8aa33..f567b4141f86 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java @@ -48,7 +48,9 @@ import java.util.Optional; import java.util.UUID; import java.util.function.Supplier; +import lombok.extern.slf4j.Slf4j; +@Slf4j @Singleton public class NormalizationActivityImpl implements NormalizationActivity { @@ -148,13 +150,16 @@ public NormalizationInput generateNormalizationInput(final StandardSyncInput syn private CheckedSupplier, Exception> getLegacyWorkerFactory( final IntegrationLauncherConfig destinationLauncherConfig, final JobRunConfig jobRunConfig) { + log.error("destination DBT -> {}", destinationLauncherConfig.getSupportDBT()); + log.error("destination normalization -> {}", destinationLauncherConfig.getDockerNormalizationImage()); return () -> new DefaultNormalizationWorker( jobRunConfig.getJobId(), Math.toIntExact(jobRunConfig.getAttemptId()), NormalizationRunnerFactory.create( destinationLauncherConfig.getDockerImage(), processFactory, - NormalizationRunnerFactory.NORMALIZATION_VERSION), + NormalizationRunnerFactory.NORMALIZATION_VERSION, + destinationLauncherConfig.getDockerNormalizationImage()), workerEnvironment); } From 7caa00c44dab0926f72c25bfa2f845c03f7ed809 Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Fri, 25 Nov 2022 16:32:02 +0200 Subject: [PATCH 02/34] updated minor remarks --- .../io/airbyte/commons/temporal/TemporalClient.java | 6 +++--- .../io/airbyte/config/persistence/DbConverter.java | 6 ++++++ .../container_orchestrator/DbtJobOrchestrator.java | 2 +- .../NormalizationJobOrchestrator.java | 2 +- .../workers_models/IntegrationLauncherConfig.yaml | 4 ++-- .../activities/GenerateInputActivityImpl.java | 13 +++++++------ .../sync/DbtTransformationActivityImpl.java | 2 +- .../temporal/sync/NormalizationActivityImpl.java | 6 +++--- 8 files changed, 24 insertions(+), 17 deletions(-) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java index fb80c5973bd3..b053ecc5a524 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java @@ -390,14 +390,14 @@ public TemporalResponse submitSync(final long jobId, final i .findFirst(); final String destinationNormalizationDockerImage = optionalDestinationDefinition.map(standardDestinationDefinition -> String.format("%s:%s", standardDestinationDefinition.getNormalizationRepository(), standardDestinationDefinition.getNormalizationTag())).orElse(null); - final boolean supportDBT = optionalDestinationDefinition.isPresent() ? optionalDestinationDefinition.get().getSupportsDbt() : false; + final boolean supportDbt = optionalDestinationDefinition.isPresent() ? optionalDestinationDefinition.get().getSupportsDbt() : false; destinationLauncherConfig = new IntegrationLauncherConfig() .withJobId(String.valueOf(jobId)) .withAttemptId((long) attempt) .withDockerImage(config.getDestinationDockerImage()) .withProtocolVersion(config.getDestinationProtocolVersion()) - .withDockerNormalizationImage(destinationNormalizationDockerImage) - .withSupportDBT(supportDBT); + .withNormalizationDockerImage(destinationNormalizationDockerImage) + .withSupportDbt(supportDbt); } catch (IOException ignored) { destinationLauncherConfig = new IntegrationLauncherConfig() .withJobId(String.valueOf(jobId)) diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java index 44c3c160da8c..251972cabf33 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java @@ -174,6 +174,12 @@ public static StandardDestinationDefinition buildStandardDestinationDefinition(f : Enums.toEnum(record.get(ACTOR_DEFINITION.RELEASE_STAGE, String.class), StandardDestinationDefinition.ReleaseStage.class).orElseThrow()) .withReleaseDate(record.get(ACTOR_DEFINITION.RELEASE_DATE) == null ? null : record.get(ACTOR_DEFINITION.RELEASE_DATE).toString()) + .withNormalizationRepository(record.get(ACTOR_DEFINITION.NORMALIZATION_REPOSITORY) == null ? null + : record.get(ACTOR_DEFINITION.NORMALIZATION_REPOSITORY)) + .withNormalizationRepository(record.get(ACTOR_DEFINITION.NORMALIZATION_TAG) == null ? null + : record.get(ACTOR_DEFINITION.NORMALIZATION_TAG)) + .withSupportsDbt(record.get(ACTOR_DEFINITION.SUPPORTS_DBT) == null ? null + : record.get(ACTOR_DEFINITION.SUPPORTS_DBT)) .withResourceRequirements(record.get(ACTOR_DEFINITION.RESOURCE_REQUIREMENTS) == null ? null : Jsons.deserialize(record.get(ACTOR_DEFINITION.RESOURCE_REQUIREMENTS).data(), ActorDefinitionResourceRequirements.class)); diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/DbtJobOrchestrator.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/DbtJobOrchestrator.java index 34782b91dff3..20fd3de9e17d 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/DbtJobOrchestrator.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/DbtJobOrchestrator.java @@ -73,7 +73,7 @@ public Optional runJob() throws Exception { destinationLauncherConfig.getDockerImage(), processFactory, NormalizationRunnerFactory.NORMALIZATION_VERSION, - destinationLauncherConfig.getDockerNormalizationImage()))); + destinationLauncherConfig.getNormalizationDockerImage()))); log.info("Running dbt worker..."); final Path jobRoot = TemporalUtils.getJobRoot(configs.getWorkspaceRoot(), jobRunConfig.getJobId(), jobRunConfig.getAttemptId()); diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/NormalizationJobOrchestrator.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/NormalizationJobOrchestrator.java index 6cdd3a112aa4..eac0424d37fb 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/NormalizationJobOrchestrator.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/NormalizationJobOrchestrator.java @@ -70,7 +70,7 @@ public Optional runJob() throws Exception { destinationLauncherConfig.getDockerImage(), processFactory, NormalizationRunnerFactory.NORMALIZATION_VERSION, - destinationLauncherConfig.getDockerNormalizationImage()), + destinationLauncherConfig.getNormalizationDockerImage()), configs.getWorkerEnvironment()); log.info("Running normalization worker..."); diff --git a/airbyte-worker-models/src/main/resources/workers_models/IntegrationLauncherConfig.yaml b/airbyte-worker-models/src/main/resources/workers_models/IntegrationLauncherConfig.yaml index dcdbb219593a..ce339b01fcb2 100644 --- a/airbyte-worker-models/src/main/resources/workers_models/IntegrationLauncherConfig.yaml +++ b/airbyte-worker-models/src/main/resources/workers_models/IntegrationLauncherConfig.yaml @@ -16,9 +16,9 @@ properties: type: integer dockerImage: type: string - dockerNormalizationImage: + normalizationDockerImage: type: string - supportDBT: + supportDbt: type: boolean default: false protocolVersion: diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java index b31ad4a07b18..9c353098eeaa 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java @@ -4,10 +4,6 @@ package io.airbyte.workers.temporal.scheduling.activities; -import static io.airbyte.metrics.lib.ApmTraceConstants.ACTIVITY_TRACE_OPERATION_NAME; -import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.ATTEMPT_NUMBER_KEY; -import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ID_KEY; - import datadog.trace.api.Trace; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.temporal.TemporalWorkflowUtils; @@ -28,10 +24,15 @@ import io.airbyte.workers.WorkerConstants; import io.micronaut.context.annotation.Requires; import jakarta.inject.Singleton; + import java.util.List; import java.util.Map; import java.util.Optional; +import static io.airbyte.metrics.lib.ApmTraceConstants.ACTIVITY_TRACE_OPERATION_NAME; +import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.ATTEMPT_NUMBER_KEY; +import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ID_KEY; + @Singleton @Requires(env = WorkerMode.CONTROL_PLANE) public class GenerateInputActivityImpl implements GenerateInputActivity { @@ -104,8 +105,8 @@ public GeneratedJobInput getSyncWorkflowInput(final SyncInput input) { .withAttemptId((long) attempt) .withDockerImage(config.getDestinationDockerImage()) .withProtocolVersion(config.getDestinationProtocolVersion()) - .withDockerNormalizationImage(destinationNormalizationDockerImage) - .withSupportDBT(supportDBT); + .withNormalizationDockerImage(destinationNormalizationDockerImage) + .withSupportDbt(supportDBT); final StandardSyncInput syncInput = new StandardSyncInput() .withNamespaceDefinition(config.getNamespaceDefinition()) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java index ec0ee6f52275..0c86aadd37b1 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java @@ -147,7 +147,7 @@ private CheckedSupplier, Exception> getLegacyWork destinationLauncherConfig.getDockerImage(), processFactory, NormalizationRunnerFactory.NORMALIZATION_VERSION, - destinationLauncherConfig.getDockerNormalizationImage()))); + destinationLauncherConfig.getNormalizationDockerImage()))); } private CheckedSupplier, Exception> getContainerLauncherWorkerFactory( diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java index f567b4141f86..79b475927814 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java @@ -150,8 +150,8 @@ public NormalizationInput generateNormalizationInput(final StandardSyncInput syn private CheckedSupplier, Exception> getLegacyWorkerFactory( final IntegrationLauncherConfig destinationLauncherConfig, final JobRunConfig jobRunConfig) { - log.error("destination DBT -> {}", destinationLauncherConfig.getSupportDBT()); - log.error("destination normalization -> {}", destinationLauncherConfig.getDockerNormalizationImage()); + log.error("destination DBT -> {}", destinationLauncherConfig.getSupportDbt()); + log.error("destination normalization -> {}", destinationLauncherConfig.getNormalizationDockerImage()); return () -> new DefaultNormalizationWorker( jobRunConfig.getJobId(), Math.toIntExact(jobRunConfig.getAttemptId()), @@ -159,7 +159,7 @@ private CheckedSupplier, Except destinationLauncherConfig.getDockerImage(), processFactory, NormalizationRunnerFactory.NORMALIZATION_VERSION, - destinationLauncherConfig.getDockerNormalizationImage()), + destinationLauncherConfig.getNormalizationDockerImage()), workerEnvironment); } From 8c46fd3b766af54aeb51c684c7d661cde996ae81 Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Fri, 25 Nov 2022 17:59:04 +0200 Subject: [PATCH 03/34] updated minor remarks --- .../commons/temporal/TemporalClient.java | 50 ++++++------------- .../config/persistence/DbConverter.java | 2 +- .../activities/GenerateInputActivityImpl.java | 23 +++++---- 3 files changed, 27 insertions(+), 48 deletions(-) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java index b053ecc5a524..e16b5006322c 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java @@ -4,8 +4,6 @@ package io.airbyte.commons.temporal; -import static io.airbyte.commons.temporal.scheduling.ConnectionManagerWorkflow.NON_RUNNING_JOB_ID; - import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ByteString; import io.airbyte.commons.temporal.config.WorkerMode; @@ -23,11 +21,9 @@ import io.airbyte.config.JobGetSpecConfig; import io.airbyte.config.JobSyncConfig; import io.airbyte.config.StandardCheckConnectionInput; -import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardDiscoverCatalogInput; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; -import io.airbyte.config.init.LocalDefinitionsProvider; import io.airbyte.config.persistence.StreamResetPersistence; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; @@ -43,6 +39,12 @@ import io.temporal.serviceclient.WorkflowServiceStubs; import jakarta.inject.Named; import jakarta.inject.Singleton; +import lombok.Builder; +import lombok.Value; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.StopWatch; + import java.io.IOException; import java.nio.file.Path; import java.util.HashSet; @@ -56,11 +58,8 @@ import java.util.concurrent.TimeoutException; import java.util.function.Supplier; import java.util.stream.Collectors; -import lombok.Builder; -import lombok.Value; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.time.StopWatch; + +import static io.airbyte.commons.temporal.scheduling.ConnectionManagerWorkflow.NON_RUNNING_JOB_ID; @Slf4j @Singleton @@ -380,32 +379,11 @@ public TemporalResponse submitSync(final long jobId, final i .withDockerImage(config.getSourceDockerImage()) .withProtocolVersion(config.getSourceProtocolVersion()); - IntegrationLauncherConfig destinationLauncherConfig; - try { - LocalDefinitionsProvider provider = new LocalDefinitionsProvider(LocalDefinitionsProvider.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS); - List destinationDefinitionList = provider.getDestinationDefinitions(); - Optional optionalDestinationDefinition = destinationDefinitionList.stream() - .filter(destinationDefinition -> config.getDestinationDockerImage() - .equalsIgnoreCase(destinationDefinition.getDockerRepository() + ":" + destinationDefinition.getDockerImageTag())) - .findFirst(); - final String destinationNormalizationDockerImage = optionalDestinationDefinition.map(standardDestinationDefinition -> String.format("%s:%s", - standardDestinationDefinition.getNormalizationRepository(), standardDestinationDefinition.getNormalizationTag())).orElse(null); - final boolean supportDbt = optionalDestinationDefinition.isPresent() ? optionalDestinationDefinition.get().getSupportsDbt() : false; - destinationLauncherConfig = new IntegrationLauncherConfig() - .withJobId(String.valueOf(jobId)) - .withAttemptId((long) attempt) - .withDockerImage(config.getDestinationDockerImage()) - .withProtocolVersion(config.getDestinationProtocolVersion()) - .withNormalizationDockerImage(destinationNormalizationDockerImage) - .withSupportDbt(supportDbt); - } catch (IOException ignored) { - destinationLauncherConfig = new IntegrationLauncherConfig() - .withJobId(String.valueOf(jobId)) - .withAttemptId((long) attempt) - .withDockerImage(config.getDestinationDockerImage()) - .withProtocolVersion(config.getDestinationProtocolVersion()); - } - final IntegrationLauncherConfig finalDestinationLauncherConfig = destinationLauncherConfig; + final IntegrationLauncherConfig destinationLauncherConfig = new IntegrationLauncherConfig() + .withJobId(String.valueOf(jobId)) + .withAttemptId((long) attempt) + .withDockerImage(config.getDestinationDockerImage()) + .withProtocolVersion(config.getDestinationProtocolVersion()); final StandardSyncInput input = new StandardSyncInput() .withNamespaceDefinition(config.getNamespaceDefinition()) @@ -424,7 +402,7 @@ public TemporalResponse submitSync(final long jobId, final i () -> getWorkflowStub(SyncWorkflow.class, TemporalJobType.SYNC).run( jobRunConfig, sourceLauncherConfig, - finalDestinationLauncherConfig, + destinationLauncherConfig, input, connectionId)); } diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java index 251972cabf33..4ae0619a1257 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java @@ -176,7 +176,7 @@ public static StandardDestinationDefinition buildStandardDestinationDefinition(f : record.get(ACTOR_DEFINITION.RELEASE_DATE).toString()) .withNormalizationRepository(record.get(ACTOR_DEFINITION.NORMALIZATION_REPOSITORY) == null ? null : record.get(ACTOR_DEFINITION.NORMALIZATION_REPOSITORY)) - .withNormalizationRepository(record.get(ACTOR_DEFINITION.NORMALIZATION_TAG) == null ? null + .withNormalizationTag(record.get(ACTOR_DEFINITION.NORMALIZATION_TAG) == null ? null : record.get(ACTOR_DEFINITION.NORMALIZATION_TAG)) .withSupportsDbt(record.get(ACTOR_DEFINITION.SUPPORTS_DBT) == null ? null : record.get(ACTOR_DEFINITION.SUPPORTS_DBT)) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java index 9c353098eeaa..0fcf1dd77819 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java @@ -4,6 +4,10 @@ package io.airbyte.workers.temporal.scheduling.activities; +import static io.airbyte.metrics.lib.ApmTraceConstants.ACTIVITY_TRACE_OPERATION_NAME; +import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.ATTEMPT_NUMBER_KEY; +import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ID_KEY; + import datadog.trace.api.Trace; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.temporal.TemporalWorkflowUtils; @@ -15,7 +19,7 @@ import io.airbyte.config.ResetSourceConfiguration; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSyncInput; -import io.airbyte.config.init.LocalDefinitionsProvider; +import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.persistence.job.JobPersistence; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; @@ -24,23 +28,21 @@ import io.airbyte.workers.WorkerConstants; import io.micronaut.context.annotation.Requires; import jakarta.inject.Singleton; - import java.util.List; import java.util.Map; import java.util.Optional; -import static io.airbyte.metrics.lib.ApmTraceConstants.ACTIVITY_TRACE_OPERATION_NAME; -import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.ATTEMPT_NUMBER_KEY; -import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ID_KEY; - @Singleton @Requires(env = WorkerMode.CONTROL_PLANE) public class GenerateInputActivityImpl implements GenerateInputActivity { private final JobPersistence jobPersistence; + private final ConfigRepository configRepository; - public GenerateInputActivityImpl(final JobPersistence jobPersistence) { + public GenerateInputActivityImpl(final JobPersistence jobPersistence, + final ConfigRepository configRepository) { this.jobPersistence = jobPersistence; + this.configRepository = configRepository; } @Trace(operationName = ACTIVITY_TRACE_OPERATION_NAME) @@ -84,15 +86,14 @@ public GeneratedJobInput getSyncWorkflowInput(final SyncInput input) { final JobRunConfig jobRunConfig = TemporalWorkflowUtils.createJobRunConfig(jobId, attempt); - LocalDefinitionsProvider provider = new LocalDefinitionsProvider(LocalDefinitionsProvider.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS); - List destinationDefinitionList = provider.getDestinationDefinitions(); + List destinationDefinitionList = configRepository.listStandardDestinationDefinitions(true); Optional optionalDestinationDefinition = destinationDefinitionList.stream() .filter(destinationDefinition -> config.getDestinationDockerImage() .equalsIgnoreCase(destinationDefinition.getDockerRepository() + ":" + destinationDefinition.getDockerImageTag())) .findFirst(); final String destinationNormalizationDockerImage = optionalDestinationDefinition.map(standardDestinationDefinition -> String.format("%s:%s", standardDestinationDefinition.getNormalizationRepository(), standardDestinationDefinition.getNormalizationTag())).orElse(null); - final boolean supportDBT = optionalDestinationDefinition.isPresent() ? optionalDestinationDefinition.get().getSupportsDbt() : false; + final boolean supportDbt = optionalDestinationDefinition.isPresent() ? optionalDestinationDefinition.get().getSupportsDbt() : false; final IntegrationLauncherConfig sourceLauncherConfig = new IntegrationLauncherConfig() .withJobId(String.valueOf(jobId)) @@ -106,7 +107,7 @@ public GeneratedJobInput getSyncWorkflowInput(final SyncInput input) { .withDockerImage(config.getDestinationDockerImage()) .withProtocolVersion(config.getDestinationProtocolVersion()) .withNormalizationDockerImage(destinationNormalizationDockerImage) - .withSupportDbt(supportDBT); + .withSupportDbt(supportDbt); final StandardSyncInput syncInput = new StandardSyncInput() .withNamespaceDefinition(config.getNamespaceDefinition()) From 8c7f464b462f7a855f2cb1deee4fbe62a773d662 Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Mon, 28 Nov 2022 17:39:01 +0200 Subject: [PATCH 04/34] fixed minor remarks --- airbyte-commons-temporal/build.gradle | 1 - .../normalization/DefaultNormalizationRunner.java | 9 +-------- .../scheduling/activities/GenerateInputActivityImpl.java | 3 ++- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/airbyte-commons-temporal/build.gradle b/airbyte-commons-temporal/build.gradle index 347bddcc1db3..7e553f1c7e94 100644 --- a/airbyte-commons-temporal/build.gradle +++ b/airbyte-commons-temporal/build.gradle @@ -15,7 +15,6 @@ dependencies { implementation project(':airbyte-config:config-models') implementation project(':airbyte-config:config-persistence') - implementation project(':airbyte-config:init') implementation project(':airbyte-metrics:metrics-lib') implementation project(':airbyte-persistence:job-persistence') implementation project(':airbyte-protocol:protocol-models') diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java index ee65ace552d8..4bf0ae6c74b8 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java @@ -79,15 +79,8 @@ public DefaultNormalizationRunner(final DestinationType destinationType, public DefaultNormalizationRunner(final ProcessFactory processFactory, final String normalizationImage) { this.processFactory = processFactory; - this.destinationType = getDestinationTypeByNormalizationImageNamePart(normalizationImage); this.normalizationImageName = normalizationImage; - } - - private static DestinationType getDestinationTypeByNormalizationImageNamePart(final String normalizationImageName) { - return EnumSet.allOf(DestinationType.class).stream() - .filter(destinationType -> normalizationImageName.contains(destinationType.name().toLowerCase())) - .findFirst() - .orElseGet(null); + this.destinationType = null; } @Override diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java index 0fcf1dd77819..deac27ea031a 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java @@ -9,6 +9,7 @@ import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ID_KEY; import datadog.trace.api.Trace; +import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.temporal.TemporalWorkflowUtils; import io.airbyte.commons.temporal.config.WorkerMode; @@ -89,7 +90,7 @@ public GeneratedJobInput getSyncWorkflowInput(final SyncInput input) { List destinationDefinitionList = configRepository.listStandardDestinationDefinitions(true); Optional optionalDestinationDefinition = destinationDefinitionList.stream() .filter(destinationDefinition -> config.getDestinationDockerImage() - .equalsIgnoreCase(destinationDefinition.getDockerRepository() + ":" + destinationDefinition.getDockerImageTag())) + .equalsIgnoreCase(DockerUtils.getTaggedImageName(destinationDefinition.getDockerRepository(), destinationDefinition.getDockerImageTag()))) .findFirst(); final String destinationNormalizationDockerImage = optionalDestinationDefinition.map(standardDestinationDefinition -> String.format("%s:%s", standardDestinationDefinition.getNormalizationRepository(), standardDestinationDefinition.getNormalizationTag())).orElse(null); From 1d90e5dbff312eb99b6885c157783dd90f4b1aab Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Tue, 29 Nov 2022 13:39:54 +0200 Subject: [PATCH 05/34] added normalization data to the tests --- airbyte-api/src/main/openapi/config.yaml | 14 +++++ .../commons/temporal/TemporalClient.java | 15 +++--- .../DefaultNormalizationRunner.java | 1 - .../DestinationDefinitionsHandler.java | 9 ++++ .../DestinationDefinitionsHandlerTest.java | 51 +++++++++++++++++++ .../activities/GenerateInputActivityImpl.java | 3 +- 6 files changed, 83 insertions(+), 10 deletions(-) diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index d1745349070b..dead676405fc 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -2982,6 +2982,13 @@ components: format: uri icon: type: string + normalizationRepository: + type: string + normalizationTag: + type: string + supportDbt: + type: boolean + default: false resourceRequirements: $ref: "#/components/schemas/ActorDefinitionResourceRequirements" DestinationDefinitionUpdate: @@ -3013,6 +3020,13 @@ components: type: string dockerImageTag: type: string + normalizationRepository: + type: string + normalizationTag: + type: string + supportDbt: + type: boolean + default: false documentationUrl: type: string format: uri diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java index e16b5006322c..9abceab23c75 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java @@ -4,6 +4,8 @@ package io.airbyte.commons.temporal; +import static io.airbyte.commons.temporal.scheduling.ConnectionManagerWorkflow.NON_RUNNING_JOB_ID; + import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ByteString; import io.airbyte.commons.temporal.config.WorkerMode; @@ -39,12 +41,6 @@ import io.temporal.serviceclient.WorkflowServiceStubs; import jakarta.inject.Named; import jakarta.inject.Singleton; -import lombok.Builder; -import lombok.Value; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.time.StopWatch; - import java.io.IOException; import java.nio.file.Path; import java.util.HashSet; @@ -58,8 +54,11 @@ import java.util.concurrent.TimeoutException; import java.util.function.Supplier; import java.util.stream.Collectors; - -import static io.airbyte.commons.temporal.scheduling.ConnectionManagerWorkflow.NON_RUNNING_JOB_ID; +import lombok.Builder; +import lombok.Value; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.StopWatch; @Slf4j @Singleton diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java index 4bf0ae6c74b8..31f547a63888 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java @@ -31,7 +31,6 @@ import java.io.InputStream; import java.nio.file.Path; import java.util.Collections; -import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java index 6eda6d4e1da5..417f21d792c5 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java @@ -95,6 +95,9 @@ static DestinationDefinitionRead buildDestinationDefinitionRead(final StandardDe .dockerRepository(standardDestinationDefinition.getDockerRepository()) .dockerImageTag(standardDestinationDefinition.getDockerImageTag()) .documentationUrl(new URI(standardDestinationDefinition.getDocumentationUrl())) + .normalizationRepository(standardDestinationDefinition.getNormalizationRepository()) + .normalizationTag(standardDestinationDefinition.getNormalizationTag()) + .supportDbt(standardDestinationDefinition.getSupportsDbt()) .icon(loadIcon(standardDestinationDefinition.getIcon())) .protocolVersion(standardDestinationDefinition.getProtocolVersion()) .releaseStage(getReleaseStage(standardDestinationDefinition)) @@ -231,6 +234,9 @@ private StandardDestinationDefinition destinationDefinitionFromCreate(final Dest .withSpec(spec) .withProtocolVersion(airbyteProtocolVersion.serialize()) .withTombstone(false) + .withSupportsDbt(destinationDefCreate.getSupportDbt()) + .withNormalizationRepository(destinationDefCreate.getNormalizationRepository()) + .withNormalizationTag(destinationDefCreate.getNormalizationTag()) .withReleaseStage(StandardDestinationDefinition.ReleaseStage.CUSTOM) .withResourceRequirements(ApiPojoConverters.actorDefResourceReqsToInternal(destinationDefCreate.getResourceRequirements())); return destinationDefinition; @@ -264,6 +270,9 @@ public DestinationDefinitionRead updateDestinationDefinition(final DestinationDe .withName(currentDestination.getName()) .withDocumentationUrl(currentDestination.getDocumentationUrl()) .withIcon(currentDestination.getIcon()) + .withNormalizationRepository(currentDestination.getNormalizationRepository()) + .withNormalizationTag(currentDestination.getNormalizationTag()) + .withSupportsDbt(currentDestination.getSupportsDbt()) .withSpec(spec) .withProtocolVersion(airbyteProtocolVersion.serialize()) .withTombstone(currentDestination.getTombstone()) diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java index daad65516f51..549eb76c3ad8 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java @@ -111,6 +111,9 @@ private StandardDestinationDefinition generateDestinationDefinition() { .withTombstone(false) .withReleaseStage(StandardDestinationDefinition.ReleaseStage.ALPHA) .withReleaseDate(TODAY_DATE_STRING) + .withNormalizationTag("0.1.2") + .withNormalizationRepository("normalization-repo") + .withSupportsDbt(true) .withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(new ResourceRequirements().withCpuRequest("2"))); } @@ -128,6 +131,9 @@ void testListDestinations() throws JsonValidationException, IOException, URISynt .dockerImageTag(destinationDefinition.getDockerImageTag()) .documentationUrl(new URI(destinationDefinition.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) + .normalizationRepository(destinationDefinition.getNormalizationRepository()) + .normalizationTag(destinationDefinition.getNormalizationTag()) + .supportDbt(destinationDefinition.getSupportsDbt()) .protocolVersion(destinationDefinition.getProtocolVersion()) .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) @@ -143,6 +149,9 @@ void testListDestinations() throws JsonValidationException, IOException, URISynt .dockerImageTag(destination2.getDockerImageTag()) .documentationUrl(new URI(destination2.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destination2.getIcon())) + .normalizationRepository(destination2.getNormalizationRepository()) + .normalizationTag(destination2.getNormalizationTag()) + .supportDbt(destination2.getSupportsDbt()) .protocolVersion(destinationDefinition.getProtocolVersion()) .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) @@ -173,6 +182,9 @@ void testListDestinationDefinitionsForWorkspace() throws IOException, URISyntaxE .dockerImageTag(destinationDefinition.getDockerImageTag()) .documentationUrl(new URI(destinationDefinition.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) + .normalizationRepository(destinationDefinition.getNormalizationRepository()) + .normalizationTag(destinationDefinition.getNormalizationTag()) + .supportDbt(destinationDefinition.getSupportsDbt()) .protocolVersion(destinationDefinition.getProtocolVersion()) .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) @@ -188,6 +200,9 @@ void testListDestinationDefinitionsForWorkspace() throws IOException, URISyntaxE .dockerImageTag(destination2.getDockerImageTag()) .documentationUrl(new URI(destination2.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destination2.getIcon())) + .normalizationRepository(destination2.getNormalizationRepository()) + .normalizationTag(destination2.getNormalizationTag()) + .supportDbt(destination2.getSupportsDbt()) .protocolVersion(destinationDefinition.getProtocolVersion()) .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) @@ -221,6 +236,9 @@ void testListPrivateDestinationDefinitions() throws IOException, URISyntaxExcept .dockerImageTag(destinationDefinition.getDockerImageTag()) .documentationUrl(new URI(destinationDefinition.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) + .normalizationRepository(destinationDefinition.getNormalizationRepository()) + .normalizationTag(destinationDefinition.getNormalizationTag()) + .supportDbt(destinationDefinition.getSupportsDbt()) .protocolVersion(destinationDefinition.getProtocolVersion()) .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) @@ -236,6 +254,9 @@ void testListPrivateDestinationDefinitions() throws IOException, URISyntaxExcept .dockerImageTag(destinationDefinition.getDockerImageTag()) .documentationUrl(new URI(destinationDefinition.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) + .normalizationRepository(destinationDefinition.getNormalizationRepository()) + .normalizationTag(destinationDefinition.getNormalizationTag()) + .supportDbt(destinationDefinition.getSupportsDbt()) .protocolVersion(destinationDefinition.getProtocolVersion()) .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) @@ -272,6 +293,12 @@ void testGetDestination() throws JsonValidationException, ConfigNotFoundExceptio .dockerImageTag(destinationDefinition.getDockerImageTag()) .documentationUrl(new URI(destinationDefinition.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) + .normalizationRepository(destinationDefinition.getNormalizationRepository()) + .normalizationTag(destinationDefinition.getNormalizationTag()) + .supportDbt(destinationDefinition.getSupportsDbt()) + .normalizationRepository(destinationDefinition.getNormalizationRepository()) + .normalizationTag(destinationDefinition.getNormalizationTag()) + .supportDbt(destinationDefinition.getSupportsDbt()) .protocolVersion(destinationDefinition.getProtocolVersion()) .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) @@ -318,6 +345,12 @@ void testGetDefinitionWithGrantForWorkspace() throws JsonValidationException, Co .dockerImageTag(destinationDefinition.getDockerImageTag()) .documentationUrl(new URI(destinationDefinition.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) + .normalizationRepository(destinationDefinition.getNormalizationRepository()) + .normalizationTag(destinationDefinition.getNormalizationTag()) + .supportDbt(destinationDefinition.getSupportsDbt()) + .normalizationRepository(destinationDefinition.getNormalizationRepository()) + .normalizationTag(destinationDefinition.getNormalizationTag()) + .supportDbt(destinationDefinition.getSupportsDbt()) .protocolVersion(destinationDefinition.getProtocolVersion()) .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) @@ -353,6 +386,9 @@ void testCreateDestinationDefinition() throws URISyntaxException, IOException, J .dockerImageTag(destination.getDockerImageTag()) .documentationUrl(new URI(destination.getDocumentationUrl())) .icon(destination.getIcon()) + .normalizationRepository(destination.getNormalizationRepository()) + .normalizationTag(destination.getNormalizationTag()) + .supportDbt(destination.getSupportsDbt()) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest())) @@ -365,6 +401,9 @@ void testCreateDestinationDefinition() throws URISyntaxException, IOException, J .documentationUrl(new URI(destination.getDocumentationUrl())) .destinationDefinitionId(destination.getDestinationDefinitionId()) .icon(DestinationDefinitionsHandler.loadIcon(destination.getIcon())) + .normalizationRepository(destination.getNormalizationRepository()) + .normalizationTag(destination.getNormalizationTag()) + .supportDbt(destination.getSupportsDbt()) .protocolVersion(DEFAULT_PROTOCOL_VERSION) .releaseStage(ReleaseStage.CUSTOM) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -432,6 +471,9 @@ void testCreateCustomDestinationDefinition() throws URISyntaxException, IOExcept .dockerImageTag(destination.getDockerImageTag()) .documentationUrl(new URI(destination.getDocumentationUrl())) .icon(destination.getIcon()) + .normalizationRepository(destination.getNormalizationRepository()) + .normalizationTag(destination.getNormalizationTag()) + .supportDbt(destination.getSupportsDbt()) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest())) @@ -448,6 +490,9 @@ void testCreateCustomDestinationDefinition() throws URISyntaxException, IOExcept .documentationUrl(new URI(destination.getDocumentationUrl())) .destinationDefinitionId(destination.getDestinationDefinitionId()) .icon(DestinationDefinitionsHandler.loadIcon(destination.getIcon())) + .normalizationRepository(destination.getNormalizationRepository()) + .normalizationTag(destination.getNormalizationTag()) + .supportDbt(destination.getSupportsDbt()) .protocolVersion(DEFAULT_PROTOCOL_VERSION) .releaseStage(ReleaseStage.CUSTOM) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -613,6 +658,12 @@ void testGrantDestinationDefinitionToWorkspace() throws JsonValidationException, .dockerImageTag(destinationDefinition.getDockerImageTag()) .documentationUrl(new URI(destinationDefinition.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) + .normalizationRepository(destinationDefinition.getNormalizationRepository()) + .normalizationTag(destinationDefinition.getNormalizationTag()) + .supportDbt(destinationDefinition.getSupportsDbt()) + .normalizationRepository(destinationDefinition.getNormalizationRepository()) + .normalizationTag(destinationDefinition.getNormalizationTag()) + .supportDbt(destinationDefinition.getSupportsDbt()) .protocolVersion(destinationDefinition.getProtocolVersion()) .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java index deac27ea031a..db1a99fdf0cd 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java @@ -90,7 +90,8 @@ public GeneratedJobInput getSyncWorkflowInput(final SyncInput input) { List destinationDefinitionList = configRepository.listStandardDestinationDefinitions(true); Optional optionalDestinationDefinition = destinationDefinitionList.stream() .filter(destinationDefinition -> config.getDestinationDockerImage() - .equalsIgnoreCase(DockerUtils.getTaggedImageName(destinationDefinition.getDockerRepository(), destinationDefinition.getDockerImageTag()))) + .equalsIgnoreCase( + DockerUtils.getTaggedImageName(destinationDefinition.getDockerRepository(), destinationDefinition.getDockerImageTag()))) .findFirst(); final String destinationNormalizationDockerImage = optionalDestinationDefinition.map(standardDestinationDefinition -> String.format("%s:%s", standardDestinationDefinition.getNormalizationRepository(), standardDestinationDefinition.getNormalizationTag())).orElse(null); From 7f97179eba6444fa1693a28c0a1c1b0dab01416c Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Tue, 29 Nov 2022 14:42:34 +0200 Subject: [PATCH 06/34] fixed minor remarks --- .../handlers/DestinationDefinitionsHandlerTest.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java index 549eb76c3ad8..c63d4ed8e243 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java @@ -296,9 +296,6 @@ void testGetDestination() throws JsonValidationException, ConfigNotFoundExceptio .normalizationRepository(destinationDefinition.getNormalizationRepository()) .normalizationTag(destinationDefinition.getNormalizationTag()) .supportDbt(destinationDefinition.getSupportsDbt()) - .normalizationRepository(destinationDefinition.getNormalizationRepository()) - .normalizationTag(destinationDefinition.getNormalizationTag()) - .supportDbt(destinationDefinition.getSupportsDbt()) .protocolVersion(destinationDefinition.getProtocolVersion()) .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) @@ -348,9 +345,6 @@ void testGetDefinitionWithGrantForWorkspace() throws JsonValidationException, Co .normalizationRepository(destinationDefinition.getNormalizationRepository()) .normalizationTag(destinationDefinition.getNormalizationTag()) .supportDbt(destinationDefinition.getSupportsDbt()) - .normalizationRepository(destinationDefinition.getNormalizationRepository()) - .normalizationTag(destinationDefinition.getNormalizationTag()) - .supportDbt(destinationDefinition.getSupportsDbt()) .protocolVersion(destinationDefinition.getProtocolVersion()) .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) @@ -661,9 +655,6 @@ void testGrantDestinationDefinitionToWorkspace() throws JsonValidationException, .normalizationRepository(destinationDefinition.getNormalizationRepository()) .normalizationTag(destinationDefinition.getNormalizationTag()) .supportDbt(destinationDefinition.getSupportsDbt()) - .normalizationRepository(destinationDefinition.getNormalizationRepository()) - .normalizationTag(destinationDefinition.getNormalizationTag()) - .supportDbt(destinationDefinition.getSupportsDbt()) .protocolVersion(destinationDefinition.getProtocolVersion()) .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) From 6360b5592266aeabad8b1d10bce6d0e60ccec3eb Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Tue, 29 Nov 2022 21:03:22 +0200 Subject: [PATCH 07/34] removed NormalizationRunnerFactory --- .../DefaultNormalizationRunner.java | 45 ++-- .../NormalizationRunnerFactory.java | 72 ------ .../DefaultNormalizationRunnerTest.java | 35 +-- .../NormalizationRunnerFactoryTest.java | 41 ---- .../DbtJobOrchestrator.java | 6 +- .../NormalizationJobOrchestrator.java | 6 +- .../standard-destination-test/build.gradle | 2 + .../DestinationAcceptanceTest.java | 29 +-- .../src/main/resources/spec.json | 1 - .../mysql/MySQLDestinationAcceptanceTest.java | 5 + .../job/errorreporter/JobErrorReporter.java | 13 +- .../errorreporter/JobErrorReporterTest.java | 2 +- .../java/io/airbyte/server/ServerApp.java | 3 - .../config/JobErrorReportingBeanFactory.java | 3 - .../sync/DbtTransformationActivityImpl.java | 6 +- .../sync/NormalizationActivityImpl.java | 6 +- .../api/generated-api-html/index.html | 214 +++++++++++------- 17 files changed, 195 insertions(+), 294 deletions(-) delete mode 100644 airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java delete mode 100644 airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/NormalizationRunnerFactoryTest.java diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java index 31f547a63888..858a28e169cb 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java @@ -5,7 +5,6 @@ package io.airbyte.workers.normalization; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.io.IOs; @@ -46,7 +45,7 @@ public class DefaultNormalizationRunner implements NormalizationRunner { .setLogPrefix("normalization") .setPrefixColor(Color.GREEN_BACKGROUND); - private final DestinationType destinationType; + private final String destinationType; private final ProcessFactory processFactory; private final String normalizationImageName; private final NormalizationAirbyteStreamFactory streamFactory = new NormalizationAirbyteStreamFactory(CONTAINER_LOG_MDC_BUILDER); @@ -55,31 +54,20 @@ public class DefaultNormalizationRunner implements NormalizationRunner { private Process process = null; - public enum DestinationType { - BIGQUERY, - MSSQL, - MYSQL, - ORACLE, - POSTGRES, - REDSHIFT, - SNOWFLAKE, - CLICKHOUSE, - TIDB - } - - public DefaultNormalizationRunner(final DestinationType destinationType, - final ProcessFactory processFactory, - final String normalizationImageName) { - this.destinationType = destinationType; - this.processFactory = processFactory; - this.normalizationImageName = normalizationImageName; - } - public DefaultNormalizationRunner(final ProcessFactory processFactory, final String normalizationImage) { this.processFactory = processFactory; this.normalizationImageName = normalizationImage; - this.destinationType = null; + this.destinationType = getDestinationTypeFromNormalizationImageName(normalizationImage); + } + + private String getDestinationTypeFromNormalizationImageName(final String normalizationImage) { + final String imageNameWithoutTag = normalizationImage.contains(":") ? normalizationImage.split(":")[0] : normalizationImage; + if ("airbyte/normalization".equalsIgnoreCase(imageNameWithoutTag)) { + return "normalization"; + } else { + return imageNameWithoutTag.replace("airbyte/normalization-", ""); + } } @Override @@ -99,12 +87,12 @@ public boolean configureDbt(final String jobId, final String gitRepoBranch = dbtConfig.getGitRepoBranch(); if (Strings.isNullOrEmpty(gitRepoBranch)) { return runProcess(jobId, attempt, jobRoot, files, resourceRequirements, "configure-dbt", - "--integration-type", destinationType.toString().toLowerCase(), + "--integration-type", destinationType.toLowerCase(), "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME, "--git-repo", gitRepoUrl); } else { return runProcess(jobId, attempt, jobRoot, files, resourceRequirements, "configure-dbt", - "--integration-type", destinationType.toString().toLowerCase(), + "--integration-type", destinationType.toLowerCase(), "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME, "--git-repo", gitRepoUrl, "--git-branch", gitRepoBranch); @@ -124,7 +112,7 @@ public boolean normalize(final String jobId, WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME, Jsons.serialize(catalog)); return runProcess(jobId, attempt, jobRoot, files, resourceRequirements, "run", - "--integration-type", destinationType.toString().toLowerCase(), + "--integration-type", destinationType.toLowerCase(), "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME, "--catalog", WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME); } @@ -221,9 +209,4 @@ private String buildInternalErrorMessageFromDbtStackTrace() { return errorMap.get(SentryExceptionHelper.ERROR_MAP_KEYS.ERROR_MAP_MESSAGE_KEY); } - @VisibleForTesting - DestinationType getDestinationType() { - return destinationType; - } - } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java deleted file mode 100644 index abf22541318d..000000000000 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.workers.normalization; - -import com.google.common.collect.ImmutableMap; -import io.airbyte.workers.normalization.DefaultNormalizationRunner.DestinationType; -import io.airbyte.workers.process.ProcessFactory; -import java.util.Map; -import java.util.Objects; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.ImmutablePair; - -@Slf4j -public class NormalizationRunnerFactory { - - public static final String BASE_NORMALIZATION_IMAGE_NAME = "airbyte/normalization"; - public static final String NORMALIZATION_VERSION = "0.2.24"; - - static final Map> NORMALIZATION_MAPPING = - ImmutableMap.>builder() - // map destination connectors (alphabetically) to their expected normalization settings - .put("airbyte/destination-bigquery", ImmutablePair.of(BASE_NORMALIZATION_IMAGE_NAME, DefaultNormalizationRunner.DestinationType.BIGQUERY)) - .put("airbyte/destination-bigquery-denormalized", - ImmutablePair.of(BASE_NORMALIZATION_IMAGE_NAME, DefaultNormalizationRunner.DestinationType.BIGQUERY)) - .put("airbyte/destination-clickhouse", ImmutablePair.of("airbyte/normalization-clickhouse", DestinationType.CLICKHOUSE)) - .put("airbyte/destination-clickhouse-strict-encrypt", ImmutablePair.of("airbyte/normalization-clickhouse", DestinationType.CLICKHOUSE)) - .put("airbyte/destination-mssql", ImmutablePair.of("airbyte/normalization-mssql", DestinationType.MSSQL)) - .put("airbyte/destination-mssql-strict-encrypt", ImmutablePair.of("airbyte/normalization-mssql", DestinationType.MSSQL)) - .put("airbyte/destination-mysql", ImmutablePair.of("airbyte/normalization-mysql", DestinationType.MYSQL)) - .put("airbyte/destination-mysql-strict-encrypt", ImmutablePair.of("airbyte/normalization-mysql", DestinationType.MYSQL)) - .put("airbyte/destination-oracle", ImmutablePair.of("airbyte/normalization-oracle", DestinationType.ORACLE)) - .put("airbyte/destination-oracle-strict-encrypt", ImmutablePair.of("airbyte/normalization-oracle", DestinationType.ORACLE)) - .put("airbyte/destination-postgres", ImmutablePair.of(BASE_NORMALIZATION_IMAGE_NAME, DestinationType.POSTGRES)) - .put("airbyte/destination-postgres-strict-encrypt", ImmutablePair.of(BASE_NORMALIZATION_IMAGE_NAME, DestinationType.POSTGRES)) - .put("airbyte/destination-redshift", ImmutablePair.of("airbyte/normalization-redshift", DestinationType.REDSHIFT)) - .put("airbyte/destination-snowflake", ImmutablePair.of("airbyte/normalization-snowflake", DestinationType.SNOWFLAKE)) - .put("airbyte/destination-tidb", ImmutablePair.of("airbyte/normalization-tidb", DestinationType.TIDB)) - .build(); - - public static NormalizationRunner create(final String connectorImageName, - final ProcessFactory processFactory, - final String normalizationVersion, - final String normalizationImage) { - final var valuePair = getNormalizationInfoForConnector(connectorImageName); - final String factoryNormalizationImage = String.format("%s:%s", valuePair.getLeft(), normalizationVersion); - log.error("destination normalization runner factory -> {}", factoryNormalizationImage); - if (Objects.nonNull(normalizationImage) - && !normalizationImage.equalsIgnoreCase(factoryNormalizationImage)) { - log.error( - "The normalization image name or tag in the definition file is different from the normalization image or tag in the NormalizationRunnerFactory!"); - log.error( - "the definition file value - {}, the NormalizationRunnerFactory value - {}", normalizationImage, factoryNormalizationImage); - } - return new DefaultNormalizationRunner( - valuePair.getRight(), - processFactory, - factoryNormalizationImage); - } - - public static ImmutablePair getNormalizationInfoForConnector(final String connectorImageName) { - final String imageNameWithoutTag = connectorImageName.contains(":") ? connectorImageName.split(":")[0] : connectorImageName; - if (NORMALIZATION_MAPPING.containsKey(imageNameWithoutTag)) { - return NORMALIZATION_MAPPING.get(imageNameWithoutTag); - } else { - throw new IllegalStateException( - String.format("Requested normalization for %s, but it is not included in the normalization mappings.", connectorImageName)); - } - } - -} diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java index 6b182f2c656e..dbaf63d7ebd4 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java @@ -15,6 +15,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.logging.LoggingHelper.Color; @@ -26,7 +27,6 @@ import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.exception.WorkerException; -import io.airbyte.workers.normalization.DefaultNormalizationRunner.DestinationType; import io.airbyte.workers.process.AirbyteIntegrationLauncher; import io.airbyte.workers.process.ProcessFactory; import java.io.ByteArrayInputStream; @@ -49,6 +49,9 @@ class DefaultNormalizationRunnerTest { private static final String JOB_ID = "0"; private static final int JOB_ATTEMPT = 0; + private static final String NORMALIZATION_IMAGE = "airbyte/normalization"; + private static final String NORMALIZATION_TAG = "0.2.24"; + private static Path logJobRoot; static { @@ -82,7 +85,7 @@ void setup() throws IOException, WorkerException { WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME, Jsons.serialize(catalog)); when(processFactory.create(AirbyteIntegrationLauncher.NORMALIZE_STEP, JOB_ID, JOB_ATTEMPT, jobRoot, - NormalizationRunnerFactory.BASE_NORMALIZATION_IMAGE_NAME, false, files, null, + "airbyte/normalization", false, files, null, workerConfigs.getResourceRequirements(), Map.of(AirbyteIntegrationLauncher.JOB_TYPE, AirbyteIntegrationLauncher.SYNC_JOB, AirbyteIntegrationLauncher.SYNC_STEP, AirbyteIntegrationLauncher.NORMALIZE_STEP), @@ -109,9 +112,7 @@ public void tearDown() throws IOException { @Test void test() throws Exception { - final NormalizationRunner runner = - new DefaultNormalizationRunner(DestinationType.BIGQUERY, processFactory, - NormalizationRunnerFactory.BASE_NORMALIZATION_IMAGE_NAME); + final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.buildImage(NORMALIZATION_IMAGE, NORMALIZATION_TAG)); when(process.exitValue()).thenReturn(0); @@ -121,9 +122,7 @@ void test() throws Exception { @Test void testLog() throws Exception { - final NormalizationRunner runner = - new DefaultNormalizationRunner(DestinationType.BIGQUERY, processFactory, - NormalizationRunnerFactory.BASE_NORMALIZATION_IMAGE_NAME); + final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.buildImage(NORMALIZATION_IMAGE, NORMALIZATION_TAG)); when(process.exitValue()).thenReturn(0); @@ -144,9 +143,7 @@ void testLog() throws Exception { void testClose() throws Exception { when(process.isAlive()).thenReturn(true).thenReturn(false); - final NormalizationRunner runner = - new DefaultNormalizationRunner(DestinationType.BIGQUERY, processFactory, - NormalizationRunnerFactory.BASE_NORMALIZATION_IMAGE_NAME); + final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.buildImage(NORMALIZATION_IMAGE, NORMALIZATION_TAG)); runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements()); runner.close(); @@ -157,9 +154,7 @@ void testClose() throws Exception { void testFailure() throws Exception { when(process.exitValue()).thenReturn(1); - final NormalizationRunner runner = - new DefaultNormalizationRunner(DestinationType.BIGQUERY, processFactory, - NormalizationRunnerFactory.BASE_NORMALIZATION_IMAGE_NAME); + final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.buildImage(NORMALIZATION_IMAGE, NORMALIZATION_TAG)); assertFalse(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements())); verify(process).waitFor(); @@ -179,9 +174,7 @@ void testFailureWithTraceMessage() throws Exception { """.replace("\n", ""); when(process.getInputStream()).thenReturn(new ByteArrayInputStream(errorTraceString.getBytes(StandardCharsets.UTF_8))); - final NormalizationRunner runner = - new DefaultNormalizationRunner(DestinationType.BIGQUERY, processFactory, - NormalizationRunnerFactory.BASE_NORMALIZATION_IMAGE_NAME); + final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.buildImage(NORMALIZATION_IMAGE, NORMALIZATION_TAG)); assertFalse(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements())); assertEquals(1, runner.getTraceMessages().count()); @@ -206,9 +199,7 @@ void testFailureWithDbtError() throws Exception { """; when(process.getInputStream()).thenReturn(new ByteArrayInputStream(dbtErrorString.getBytes(StandardCharsets.UTF_8))); - final NormalizationRunner runner = - new DefaultNormalizationRunner(DestinationType.BIGQUERY, processFactory, - NormalizationRunnerFactory.BASE_NORMALIZATION_IMAGE_NAME); + final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.buildImage(NORMALIZATION_IMAGE, NORMALIZATION_TAG)); assertFalse(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements())); assertEquals(1, runner.getTraceMessages().count()); @@ -228,9 +219,7 @@ void testFailureWithDbtErrorJsonFormat() throws Exception { """; when(process.getInputStream()).thenReturn(new ByteArrayInputStream(dbtErrorString.getBytes(StandardCharsets.UTF_8))); - final NormalizationRunner runner = - new DefaultNormalizationRunner(DestinationType.BIGQUERY, processFactory, - NormalizationRunnerFactory.BASE_NORMALIZATION_IMAGE_NAME); + final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.buildImage(NORMALIZATION_IMAGE, NORMALIZATION_TAG)); assertFalse(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements())); assertEquals(1, runner.getTraceMessages().count()); diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/NormalizationRunnerFactoryTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/NormalizationRunnerFactoryTest.java deleted file mode 100644 index bc1420c6d1a5..000000000000 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/NormalizationRunnerFactoryTest.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.workers.normalization; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.mock; - -import io.airbyte.workers.normalization.DefaultNormalizationRunner.DestinationType; -import io.airbyte.workers.process.ProcessFactory; -import java.util.Map.Entry; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class NormalizationRunnerFactoryTest { - - public static final String NORMALIZATION_VERSION = "dev"; - private ProcessFactory processFactory; - - @BeforeEach - void setup() { - processFactory = mock(ProcessFactory.class); - } - - @Test - void testMappings() { - for (final Entry> entry : NormalizationRunnerFactory.NORMALIZATION_MAPPING.entrySet()) { - assertEquals(entry.getValue().getValue(), - ((DefaultNormalizationRunner) NormalizationRunnerFactory.create( - String.format("%s:0.1.0", entry.getKey()), processFactory, NORMALIZATION_VERSION, String.format("%s:0.1.0", entry.getKey()))) - .getDestinationType()); - } - assertThrows(IllegalStateException.class, - () -> NormalizationRunnerFactory.create("airbyte/destination-csv:0.1.0", processFactory, - NORMALIZATION_VERSION, "airbyte/destination-csv:0.1.0")); - } - -} diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/DbtJobOrchestrator.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/DbtJobOrchestrator.java index 20fd3de9e17d..d1f7ebc3f82f 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/DbtJobOrchestrator.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/DbtJobOrchestrator.java @@ -18,7 +18,7 @@ import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.general.DbtTransformationRunner; import io.airbyte.workers.general.DbtTransformationWorker; -import io.airbyte.workers.normalization.NormalizationRunnerFactory; +import io.airbyte.workers.normalization.DefaultNormalizationRunner; import io.airbyte.workers.process.KubePodProcess; import io.airbyte.workers.process.ProcessFactory; import io.airbyte.workers.sync.ReplicationLauncherWorker; @@ -69,10 +69,8 @@ public Optional runJob() throws Exception { Math.toIntExact(jobRunConfig.getAttemptId()), workerConfigs.getResourceRequirements(), new DbtTransformationRunner( - processFactory, NormalizationRunnerFactory.create( - destinationLauncherConfig.getDockerImage(), + processFactory, new DefaultNormalizationRunner( processFactory, - NormalizationRunnerFactory.NORMALIZATION_VERSION, destinationLauncherConfig.getNormalizationDockerImage()))); log.info("Running dbt worker..."); diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/NormalizationJobOrchestrator.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/NormalizationJobOrchestrator.java index eac0424d37fb..5b5f26e70cb6 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/NormalizationJobOrchestrator.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/NormalizationJobOrchestrator.java @@ -18,7 +18,7 @@ import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.workers.general.DefaultNormalizationWorker; -import io.airbyte.workers.normalization.NormalizationRunnerFactory; +import io.airbyte.workers.normalization.DefaultNormalizationRunner; import io.airbyte.workers.normalization.NormalizationWorker; import io.airbyte.workers.process.KubePodProcess; import io.airbyte.workers.process.ProcessFactory; @@ -66,10 +66,8 @@ public Optional runJob() throws Exception { final NormalizationWorker normalizationWorker = new DefaultNormalizationWorker( jobRunConfig.getJobId(), Math.toIntExact(jobRunConfig.getAttemptId()), - NormalizationRunnerFactory.create( - destinationLauncherConfig.getDockerImage(), + new DefaultNormalizationRunner( processFactory, - NormalizationRunnerFactory.NORMALIZATION_VERSION, destinationLauncherConfig.getNormalizationDockerImage()), configs.getWorkerEnvironment()); diff --git a/airbyte-integrations/bases/standard-destination-test/build.gradle b/airbyte-integrations/bases/standard-destination-test/build.gradle index d69a666b31fa..48568f05ace5 100644 --- a/airbyte-integrations/bases/standard-destination-test/build.gradle +++ b/airbyte-integrations/bases/standard-destination-test/build.gradle @@ -5,6 +5,8 @@ dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-commons-worker') implementation project(':airbyte-config:config-models') + implementation project(':airbyte-config:config-persistence') + implementation project(':airbyte-json-validation') implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-protocol:protocol-models') implementation project(':airbyte-commons-worker') diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java index 4e077a96eadd..d43f593bfab3 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -57,8 +57,8 @@ import io.airbyte.workers.helper.EntrypointEnvChecker; import io.airbyte.workers.internal.AirbyteDestination; import io.airbyte.workers.internal.DefaultAirbyteDestination; +import io.airbyte.workers.normalization.DefaultNormalizationRunner; import io.airbyte.workers.normalization.NormalizationRunner; -import io.airbyte.workers.normalization.NormalizationRunnerFactory; import io.airbyte.workers.process.AirbyteIntegrationLauncher; import io.airbyte.workers.process.DockerProcessFactory; import io.airbyte.workers.process.ProcessFactory; @@ -121,6 +121,10 @@ public abstract class DestinationAcceptanceTest { */ protected abstract String getImageName(); + protected String getNormalizationImageName() { + return null; + } + /** * Configuration specific to the integration. Will be passed to integration where appropriate in * each test. Should be valid. @@ -542,7 +546,9 @@ public void specNormalizationValueShouldBeCorrect() throws Exception { if (normalizationFromSpec) { boolean normalizationRunnerFactorySupportsDestinationImage; try { - NormalizationRunnerFactory.create(getImageName(), processFactory, NORMALIZATION_VERSION, ""); + new DefaultNormalizationRunner( + processFactory, + getNormalizationImageName()); normalizationRunnerFactorySupportsDestinationImage = true; } catch (final IllegalStateException e) { normalizationRunnerFactorySupportsDestinationImage = false; @@ -845,10 +851,9 @@ public void testCustomDbtTransformations() throws Exception { // 'profiles.yml' // (we don't actually rely on normalization running anything else here though) final DbtTransformationRunner runner = new DbtTransformationRunner(processFactory, - NormalizationRunnerFactory.create( - getImageName(), + new DefaultNormalizationRunner( processFactory, - NORMALIZATION_VERSION, "")); + getNormalizationImageName())); runner.start(); final Path transformationRoot = Files.createDirectories(jobRoot.resolve("transform")); final OperatorDbt dbtConfig = new OperatorDbt() @@ -861,9 +866,7 @@ public void testCustomDbtTransformations() throws Exception { // TODO once we're on DBT 1.x, switch this back to using the main branch .withGitRepoUrl("https://github.com/airbytehq/jaffle_shop.git") .withGitRepoBranch("pre_dbt_upgrade") - .withDockerImage( - NormalizationRunnerFactory.getNormalizationInfoForConnector(getImageName()).getLeft() - + ":" + NORMALIZATION_VERSION); + .withDockerImage(getNormalizationImageName()); // // jaffle_shop is a fictional ecommerce store maintained by fishtownanalytics/dbt. // @@ -923,10 +926,9 @@ void testCustomDbtTransformationsFailure() throws Exception { final JsonNode config = getConfig(); final DbtTransformationRunner runner = new DbtTransformationRunner(processFactory, - NormalizationRunnerFactory.create( - getImageName(), + new DefaultNormalizationRunner( processFactory, - NORMALIZATION_VERSION, "")); + getNormalizationImageName())); runner.start(); final Path transformationRoot = Files.createDirectories(jobRoot.resolve("transform")); final OperatorDbt dbtConfig = new OperatorDbt() @@ -1270,10 +1272,9 @@ private List runSync( return destinationOutput; } - final NormalizationRunner runner = NormalizationRunnerFactory.create( - getImageName(), + final NormalizationRunner runner = new DefaultNormalizationRunner( processFactory, - NORMALIZATION_VERSION, ""); + getNormalizationImageName()); runner.start(); final Path normalizationRoot = Files.createDirectories(jobRoot.resolve("normalize")); if (!runner.normalize(JOB_ID, JOB_ATTEMPT, normalizationRoot, diff --git a/airbyte-integrations/connectors/destination-mysql/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-mysql/src/main/resources/spec.json index 7b068ddc74e6..080158fc140e 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-mysql/src/main/resources/spec.json @@ -2,7 +2,6 @@ "documentationUrl": "https://docs.airbyte.com/integrations/destinations/mysql", "supportsIncremental": true, "supportsNormalization": true, - "supportsDBT": true, "supported_destination_sync_modes": ["overwrite", "append"], "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java index b720ea006c3c..13fc7e2bad5c 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java @@ -50,6 +50,11 @@ protected String getImageName() { return "airbyte/destination-mysql:dev"; } + @Override + protected String getNormalizationImageName() { + return "airbyte/normalization-mysql:0.2.24"; + } + @Override protected boolean supportsDBT() { return true; diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/JobErrorReporter.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/JobErrorReporter.java index 64d9996bf3d3..a931e97f1274 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/JobErrorReporter.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/JobErrorReporter.java @@ -55,24 +55,18 @@ public class JobErrorReporter { private final ConfigRepository configRepository; private final DeploymentMode deploymentMode; private final String airbyteVersion; - private final String normalizationImage; - private final String normalizationVersion; private final WebUrlHelper webUrlHelper; private final JobErrorReportingClient jobErrorReportingClient; public JobErrorReporter(final ConfigRepository configRepository, final DeploymentMode deploymentMode, final String airbyteVersion, - final String normalizationImage, - final String normalizationVersion, final WebUrlHelper webUrlHelper, final JobErrorReportingClient jobErrorReportingClient) { this.configRepository = configRepository; this.deploymentMode = deploymentMode; this.airbyteVersion = airbyteVersion; - this.normalizationImage = normalizationImage; - this.normalizationVersion = normalizationVersion; this.webUrlHelper = webUrlHelper; this.jobErrorReportingClient = jobErrorReportingClient; } @@ -119,10 +113,11 @@ public void reportSyncJobFailure(final UUID connectionId, final AttemptFailureSu // the destination) final Map metadata = MoreMaps.merge( commonMetadata, - getNormalizationMetadata(), + getNormalizationMetadata(destinationDefinition.getNormalizationRepository()), prefixConnectorMetadataKeys(getSourceMetadata(sourceDefinition), "source"), getDestinationMetadata(destinationDefinition)); - final String dockerImage = String.format("%s:%s", normalizationImage, normalizationVersion); + final String dockerImage = String.format("%s:%s", destinationDefinition.getNormalizationRepository(), + destinationDefinition.getNormalizationTag()); reportJobFailureReason(workspace, failureReason, dockerImage, metadata); } @@ -229,7 +224,7 @@ private Map getSourceMetadata(final StandardSourceDefinition sou Map.entry(CONNECTOR_RELEASE_STAGE_META_KEY, sourceDefinition.getReleaseStage().value())); } - private Map getNormalizationMetadata() { + private Map getNormalizationMetadata(String normalizationImage) { return Map.ofEntries( Map.entry(NORMALIZATION_REPOSITORY_META_KEY, normalizationImage)); } diff --git a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/JobErrorReporterTest.java b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/JobErrorReporterTest.java index ae97e8fbeedb..f47b2996adad 100644 --- a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/JobErrorReporterTest.java +++ b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/JobErrorReporterTest.java @@ -85,7 +85,7 @@ void setup() { jobErrorReportingClient = mock(JobErrorReportingClient.class); webUrlHelper = mock(WebUrlHelper.class); jobErrorReporter = new JobErrorReporter( - configRepository, DEPLOYMENT_MODE, AIRBYTE_VERSION, NORMALIZATION_IMAGE, NORMALIZATION_VERSION, webUrlHelper, jobErrorReportingClient); + configRepository, DEPLOYMENT_MODE, AIRBYTE_VERSION, webUrlHelper, jobErrorReportingClient); Mockito.when(webUrlHelper.getConnectionUrl(WORKSPACE_ID, CONNECTION_ID)).thenReturn(CONNECTION_URL); Mockito.when(webUrlHelper.getWorkspaceUrl(WORKSPACE_ID)).thenReturn(WORKSPACE_URL); diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index e74c91094418..10b464cb247e 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -71,7 +71,6 @@ import io.airbyte.server.scheduler.TemporalEventRunner; import io.airbyte.validation.json.JsonSchemaValidator; import io.airbyte.workers.helper.ConnectionHelper; -import io.airbyte.workers.normalization.NormalizationRunnerFactory; import io.temporal.serviceclient.WorkflowServiceStubs; import java.net.http.HttpClient; import java.util.Map; @@ -220,8 +219,6 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, configRepository, configs.getDeploymentMode(), configs.getAirbyteVersionOrWarning(), - NormalizationRunnerFactory.BASE_NORMALIZATION_IMAGE_NAME, - NormalizationRunnerFactory.NORMALIZATION_VERSION, webUrlHelper, jobErrorReportingClient); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/config/JobErrorReportingBeanFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/config/JobErrorReportingBeanFactory.java index f9f6c8aeb03d..44b1e323408f 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/config/JobErrorReportingBeanFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/config/JobErrorReportingBeanFactory.java @@ -13,7 +13,6 @@ import io.airbyte.persistence.job.errorreporter.LoggingJobErrorReportingClient; import io.airbyte.persistence.job.errorreporter.SentryExceptionHelper; import io.airbyte.persistence.job.errorreporter.SentryJobErrorReportingClient; -import io.airbyte.workers.normalization.NormalizationRunnerFactory; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Requires; import io.micronaut.context.annotation.Value; @@ -59,8 +58,6 @@ public JobErrorReporter jobErrorReporter( configRepository, deploymentMode, airbyteVersion, - NormalizationRunnerFactory.BASE_NORMALIZATION_IMAGE_NAME, - NormalizationRunnerFactory.NORMALIZATION_VERSION, webUrlHelper, jobErrorReportingClient.orElseGet(() -> new LoggingJobErrorReportingClient())); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java index 447174e223cf..9ad830e05378 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java @@ -31,7 +31,7 @@ import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.general.DbtTransformationRunner; import io.airbyte.workers.general.DbtTransformationWorker; -import io.airbyte.workers.normalization.NormalizationRunnerFactory; +import io.airbyte.workers.normalization.DefaultNormalizationRunner; import io.airbyte.workers.process.ProcessFactory; import io.airbyte.workers.sync.DbtLauncherWorker; import io.airbyte.workers.temporal.TemporalAttemptExecution; @@ -142,10 +142,8 @@ private CheckedSupplier, Exception> getLegacyWork Math.toIntExact(jobRunConfig.getAttemptId()), resourceRequirements, new DbtTransformationRunner( - processFactory, NormalizationRunnerFactory.create( - destinationLauncherConfig.getDockerImage(), + processFactory, new DefaultNormalizationRunner( processFactory, - NormalizationRunnerFactory.NORMALIZATION_VERSION, destinationLauncherConfig.getNormalizationDockerImage()))); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java index 88636e539fef..feb63e6668ec 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java @@ -33,7 +33,7 @@ import io.airbyte.workers.Worker; import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.general.DefaultNormalizationWorker; -import io.airbyte.workers.normalization.NormalizationRunnerFactory; +import io.airbyte.workers.normalization.DefaultNormalizationRunner; import io.airbyte.workers.process.ProcessFactory; import io.airbyte.workers.sync.NormalizationLauncherWorker; import io.airbyte.workers.temporal.TemporalAttemptExecution; @@ -154,10 +154,8 @@ private CheckedSupplier, Except return () -> new DefaultNormalizationWorker( jobRunConfig.getJobId(), Math.toIntExact(jobRunConfig.getAttemptId()), - NormalizationRunnerFactory.create( - destinationLauncherConfig.getDockerImage(), + new DefaultNormalizationRunner( processFactory, - NormalizationRunnerFactory.NORMALIZATION_VERSION, destinationLauncherConfig.getNormalizationDockerImage()), workerEnvironment); } diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index 24555ce4519d..3eb23b6841fe 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -2617,6 +2617,12 @@

Return type

Example data

Content-Type: application/json
{
+  "documentationUrl" : "https://openapi-generator.tech",
+  "releaseDate" : "2000-01-23",
+  "dockerRepository" : "dockerRepository",
+  "normalizationRepository" : "normalizationRepository",
+  "icon" : "icon",
+  "supportDbt" : false,
   "resourceRequirements" : {
     "default" : {
       "cpu_limit" : "cpu_limit",
@@ -2640,14 +2646,11 @@ 
Example data

       }
     } ]
   },
-  "documentationUrl" : "https://openapi-generator.tech",
   "dockerImageTag" : "dockerImageTag",
-  "releaseDate" : "2000-01-23",
-  "dockerRepository" : "dockerRepository",
   "name" : "name",
-  "icon" : "icon",
   "protocolVersion" : "protocolVersion",
-  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "normalizationTag" : "normalizationTag"
 }

Produces

@@ -2702,6 +2705,12 @@

Return type

Example data

Content-Type: application/json
{
+  "documentationUrl" : "https://openapi-generator.tech",
+  "releaseDate" : "2000-01-23",
+  "dockerRepository" : "dockerRepository",
+  "normalizationRepository" : "normalizationRepository",
+  "icon" : "icon",
+  "supportDbt" : false,
   "resourceRequirements" : {
     "default" : {
       "cpu_limit" : "cpu_limit",
@@ -2725,14 +2734,11 @@ 
Example data

       }
     } ]
   },
-  "documentationUrl" : "https://openapi-generator.tech",
   "dockerImageTag" : "dockerImageTag",
-  "releaseDate" : "2000-01-23",
-  "dockerRepository" : "dockerRepository",
   "name" : "name",
-  "icon" : "icon",
   "protocolVersion" : "protocolVersion",
-  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "normalizationTag" : "normalizationTag"
 }

Produces

@@ -2883,6 +2889,12 @@

Return type

Example data

Content-Type: application/json
{
+  "documentationUrl" : "https://openapi-generator.tech",
+  "releaseDate" : "2000-01-23",
+  "dockerRepository" : "dockerRepository",
+  "normalizationRepository" : "normalizationRepository",
+  "icon" : "icon",
+  "supportDbt" : false,
   "resourceRequirements" : {
     "default" : {
       "cpu_limit" : "cpu_limit",
@@ -2906,14 +2918,11 @@ 
Example data

       }
     } ]
   },
-  "documentationUrl" : "https://openapi-generator.tech",
   "dockerImageTag" : "dockerImageTag",
-  "releaseDate" : "2000-01-23",
-  "dockerRepository" : "dockerRepository",
   "name" : "name",
-  "icon" : "icon",
   "protocolVersion" : "protocolVersion",
-  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "normalizationTag" : "normalizationTag"
 }

Produces

@@ -2971,6 +2980,12 @@

Return type

Example data

Content-Type: application/json
{
+  "documentationUrl" : "https://openapi-generator.tech",
+  "releaseDate" : "2000-01-23",
+  "dockerRepository" : "dockerRepository",
+  "normalizationRepository" : "normalizationRepository",
+  "icon" : "icon",
+  "supportDbt" : false,
   "resourceRequirements" : {
     "default" : {
       "cpu_limit" : "cpu_limit",
@@ -2994,14 +3009,11 @@ 
Example data

       }
     } ]
   },
-  "documentationUrl" : "https://openapi-generator.tech",
   "dockerImageTag" : "dockerImageTag",
-  "releaseDate" : "2000-01-23",
-  "dockerRepository" : "dockerRepository",
   "name" : "name",
-  "icon" : "icon",
   "protocolVersion" : "protocolVersion",
-  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "normalizationTag" : "normalizationTag"
 }

Produces

@@ -3060,6 +3072,12 @@

Example data

Content-Type: application/json
{
   "destinationDefinition" : {
+    "documentationUrl" : "https://openapi-generator.tech",
+    "releaseDate" : "2000-01-23",
+    "dockerRepository" : "dockerRepository",
+    "normalizationRepository" : "normalizationRepository",
+    "icon" : "icon",
+    "supportDbt" : false,
     "resourceRequirements" : {
       "default" : {
         "cpu_limit" : "cpu_limit",
@@ -3083,14 +3101,11 @@ 
Example data

         }
       } ]
     },
-    "documentationUrl" : "https://openapi-generator.tech",
     "dockerImageTag" : "dockerImageTag",
-    "releaseDate" : "2000-01-23",
-    "dockerRepository" : "dockerRepository",
     "name" : "name",
-    "icon" : "icon",
     "protocolVersion" : "protocolVersion",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "normalizationTag" : "normalizationTag"
   },
   "granted" : true
 }
@@ -3139,6 +3154,12 @@

Example data

Content-Type: application/json
{
   "destinationDefinitions" : [ {
+    "documentationUrl" : "https://openapi-generator.tech",
+    "releaseDate" : "2000-01-23",
+    "dockerRepository" : "dockerRepository",
+    "normalizationRepository" : "normalizationRepository",
+    "icon" : "icon",
+    "supportDbt" : false,
     "resourceRequirements" : {
       "default" : {
         "cpu_limit" : "cpu_limit",
@@ -3162,15 +3183,18 @@ 
Example data

         }
       } ]
     },
-    "documentationUrl" : "https://openapi-generator.tech",
     "dockerImageTag" : "dockerImageTag",
-    "releaseDate" : "2000-01-23",
-    "dockerRepository" : "dockerRepository",
     "name" : "name",
-    "icon" : "icon",
     "protocolVersion" : "protocolVersion",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "normalizationTag" : "normalizationTag"
   }, {
+    "documentationUrl" : "https://openapi-generator.tech",
+    "releaseDate" : "2000-01-23",
+    "dockerRepository" : "dockerRepository",
+    "normalizationRepository" : "normalizationRepository",
+    "icon" : "icon",
+    "supportDbt" : false,
     "resourceRequirements" : {
       "default" : {
         "cpu_limit" : "cpu_limit",
@@ -3194,14 +3218,11 @@ 
Example data

         }
       } ]
     },
-    "documentationUrl" : "https://openapi-generator.tech",
     "dockerImageTag" : "dockerImageTag",
-    "releaseDate" : "2000-01-23",
-    "dockerRepository" : "dockerRepository",
     "name" : "name",
-    "icon" : "icon",
     "protocolVersion" : "protocolVersion",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "normalizationTag" : "normalizationTag"
   } ]
 }
@@ -3255,6 +3276,12 @@

Example data

Content-Type: application/json
{
   "destinationDefinitions" : [ {
+    "documentationUrl" : "https://openapi-generator.tech",
+    "releaseDate" : "2000-01-23",
+    "dockerRepository" : "dockerRepository",
+    "normalizationRepository" : "normalizationRepository",
+    "icon" : "icon",
+    "supportDbt" : false,
     "resourceRequirements" : {
       "default" : {
         "cpu_limit" : "cpu_limit",
@@ -3278,15 +3305,18 @@ 
Example data

         }
       } ]
     },
-    "documentationUrl" : "https://openapi-generator.tech",
     "dockerImageTag" : "dockerImageTag",
-    "releaseDate" : "2000-01-23",
-    "dockerRepository" : "dockerRepository",
     "name" : "name",
-    "icon" : "icon",
     "protocolVersion" : "protocolVersion",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "normalizationTag" : "normalizationTag"
   }, {
+    "documentationUrl" : "https://openapi-generator.tech",
+    "releaseDate" : "2000-01-23",
+    "dockerRepository" : "dockerRepository",
+    "normalizationRepository" : "normalizationRepository",
+    "icon" : "icon",
+    "supportDbt" : false,
     "resourceRequirements" : {
       "default" : {
         "cpu_limit" : "cpu_limit",
@@ -3310,14 +3340,11 @@ 
Example data

         }
       } ]
     },
-    "documentationUrl" : "https://openapi-generator.tech",
     "dockerImageTag" : "dockerImageTag",
-    "releaseDate" : "2000-01-23",
-    "dockerRepository" : "dockerRepository",
     "name" : "name",
-    "icon" : "icon",
     "protocolVersion" : "protocolVersion",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "normalizationTag" : "normalizationTag"
   } ]
 }
@@ -3359,6 +3386,12 @@

Example data

Content-Type: application/json
{
   "destinationDefinitions" : [ {
+    "documentationUrl" : "https://openapi-generator.tech",
+    "releaseDate" : "2000-01-23",
+    "dockerRepository" : "dockerRepository",
+    "normalizationRepository" : "normalizationRepository",
+    "icon" : "icon",
+    "supportDbt" : false,
     "resourceRequirements" : {
       "default" : {
         "cpu_limit" : "cpu_limit",
@@ -3382,15 +3415,18 @@ 
Example data

         }
       } ]
     },
-    "documentationUrl" : "https://openapi-generator.tech",
     "dockerImageTag" : "dockerImageTag",
-    "releaseDate" : "2000-01-23",
-    "dockerRepository" : "dockerRepository",
     "name" : "name",
-    "icon" : "icon",
     "protocolVersion" : "protocolVersion",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "normalizationTag" : "normalizationTag"
   }, {
+    "documentationUrl" : "https://openapi-generator.tech",
+    "releaseDate" : "2000-01-23",
+    "dockerRepository" : "dockerRepository",
+    "normalizationRepository" : "normalizationRepository",
+    "icon" : "icon",
+    "supportDbt" : false,
     "resourceRequirements" : {
       "default" : {
         "cpu_limit" : "cpu_limit",
@@ -3414,14 +3450,11 @@ 
Example data

         }
       } ]
     },
-    "documentationUrl" : "https://openapi-generator.tech",
     "dockerImageTag" : "dockerImageTag",
-    "releaseDate" : "2000-01-23",
-    "dockerRepository" : "dockerRepository",
     "name" : "name",
-    "icon" : "icon",
     "protocolVersion" : "protocolVersion",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "normalizationTag" : "normalizationTag"
   } ]
 }
@@ -3476,6 +3509,12 @@

Example data

{
   "destinationDefinitions" : [ {
     "destinationDefinition" : {
+      "documentationUrl" : "https://openapi-generator.tech",
+      "releaseDate" : "2000-01-23",
+      "dockerRepository" : "dockerRepository",
+      "normalizationRepository" : "normalizationRepository",
+      "icon" : "icon",
+      "supportDbt" : false,
       "resourceRequirements" : {
         "default" : {
           "cpu_limit" : "cpu_limit",
@@ -3499,18 +3538,21 @@ 
Example data

           }
         } ]
       },
-      "documentationUrl" : "https://openapi-generator.tech",
       "dockerImageTag" : "dockerImageTag",
-      "releaseDate" : "2000-01-23",
-      "dockerRepository" : "dockerRepository",
       "name" : "name",
-      "icon" : "icon",
       "protocolVersion" : "protocolVersion",
-      "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+      "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+      "normalizationTag" : "normalizationTag"
     },
     "granted" : true
   }, {
     "destinationDefinition" : {
+      "documentationUrl" : "https://openapi-generator.tech",
+      "releaseDate" : "2000-01-23",
+      "dockerRepository" : "dockerRepository",
+      "normalizationRepository" : "normalizationRepository",
+      "icon" : "icon",
+      "supportDbt" : false,
       "resourceRequirements" : {
         "default" : {
           "cpu_limit" : "cpu_limit",
@@ -3534,14 +3576,11 @@ 
Example data

           }
         } ]
       },
-      "documentationUrl" : "https://openapi-generator.tech",
       "dockerImageTag" : "dockerImageTag",
-      "releaseDate" : "2000-01-23",
-      "dockerRepository" : "dockerRepository",
       "name" : "name",
-      "icon" : "icon",
       "protocolVersion" : "protocolVersion",
-      "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+      "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+      "normalizationTag" : "normalizationTag"
     },
     "granted" : true
   } ]
@@ -3644,6 +3683,12 @@ 
Return type

     
Example data

     
Content-Type: application/json

     
{
+  "documentationUrl" : "https://openapi-generator.tech",
+  "releaseDate" : "2000-01-23",
+  "dockerRepository" : "dockerRepository",
+  "normalizationRepository" : "normalizationRepository",
+  "icon" : "icon",
+  "supportDbt" : false,
   "resourceRequirements" : {
     "default" : {
       "cpu_limit" : "cpu_limit",
@@ -3667,14 +3712,11 @@ 
Example data

       }
     } ]
   },
-  "documentationUrl" : "https://openapi-generator.tech",
   "dockerImageTag" : "dockerImageTag",
-  "releaseDate" : "2000-01-23",
-  "dockerRepository" : "dockerRepository",
   "name" : "name",
-  "icon" : "icon",
   "protocolVersion" : "protocolVersion",
-  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "normalizationTag" : "normalizationTag"
 }

 
     
Produces

@@ -3732,6 +3774,12 @@ 
Return type

     
Example data

     
Content-Type: application/json

     
{
+  "documentationUrl" : "https://openapi-generator.tech",
+  "releaseDate" : "2000-01-23",
+  "dockerRepository" : "dockerRepository",
+  "normalizationRepository" : "normalizationRepository",
+  "icon" : "icon",
+  "supportDbt" : false,
   "resourceRequirements" : {
     "default" : {
       "cpu_limit" : "cpu_limit",
@@ -3755,14 +3803,11 @@ 
Example data

       }
     } ]
   },
-  "documentationUrl" : "https://openapi-generator.tech",
   "dockerImageTag" : "dockerImageTag",
-  "releaseDate" : "2000-01-23",
-  "dockerRepository" : "dockerRepository",
   "name" : "name",
-  "icon" : "icon",
   "protocolVersion" : "protocolVersion",
-  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "normalizationTag" : "normalizationTag"
 }

 
     
Produces

@@ -4654,6 +4699,12 @@ 
Example data

     "airbyteVersion" : "airbyteVersion",
     "id" : 0,
     "destinationDefinition" : {
+      "documentationUrl" : "https://openapi-generator.tech",
+      "releaseDate" : "2000-01-23",
+      "dockerRepository" : "dockerRepository",
+      "normalizationRepository" : "normalizationRepository",
+      "icon" : "icon",
+      "supportDbt" : false,
       "resourceRequirements" : {
         "default" : {
           "cpu_limit" : "cpu_limit",
@@ -4677,14 +4728,11 @@ 
Example data

           }
         } ]
       },
-      "documentationUrl" : "https://openapi-generator.tech",
       "dockerImageTag" : "dockerImageTag",
-      "releaseDate" : "2000-01-23",
-      "dockerRepository" : "dockerRepository",
       "name" : "name",
-      "icon" : "icon",
       "protocolVersion" : "protocolVersion",
-      "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+      "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+      "normalizationTag" : "normalizationTag"
     }
   },
   "attempts" : [ {
@@ -11108,6 +11156,9 @@ 
DestinationDefinitionCreatedockerImageTag 
String  

 
documentationUrl 
URI  format: uri

 
icon (optional)
String  

+
normalizationRepository (optional)
String  

+
normalizationTag (optional)
String  

+
supportDbt (optional)
Boolean  

 
resourceRequirements (optional)
ActorDefinitionResourceRequirements  

       
   
@@ -11134,6 +11185,9 @@ 
DestinationDefinitionRead -
 
name 
String  

 
dockerRepository 
String  

 
dockerImageTag 
String  

+
normalizationRepository (optional)
String  

+
normalizationTag (optional)
String  

+
supportDbt (optional)
Boolean  

 
documentationUrl 
URI  format: uri

 
icon (optional)
String  

 
protocolVersion (optional)
String The Airbyte Protocol version supported by the connector 


From e04c848c87a13e9a1a5c99e87ad41e5bfc7f87c2 Mon Sep 17 00:00:00 2001
From: Andrii Korotkov 
Date: Tue, 29 Nov 2022 22:10:51 +0200
Subject: [PATCH 08/34] fixed remarks

---
 airbyte-api/src/main/openapi/config.yaml      | 14 -------
 .../NormalizationRunnerFactory.java           |  1 -
 .../DestinationDefinitionsHandler.java        |  6 ---
 .../DestinationDefinitionsHandlerTest.java    | 42 -------------------
 .../IntegrationLauncherConfig.yaml            |  2 +-
 .../activities/GenerateInputActivityImpl.java |  2 +-
 .../sync/NormalizationActivityImpl.java       |  4 --
 7 files changed, 2 insertions(+), 69 deletions(-)

diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml
index 5756ce1ccafc..cf605d97da51 100644
--- a/airbyte-api/src/main/openapi/config.yaml
+++ b/airbyte-api/src/main/openapi/config.yaml
@@ -2982,13 +2982,6 @@ components:
           format: uri
         icon:
           type: string
-        normalizationRepository:
-          type: string
-        normalizationTag:
-          type: string
-        supportDbt:
-          type: boolean
-          default: false
         resourceRequirements:
           $ref: "#/components/schemas/ActorDefinitionResourceRequirements"
     DestinationDefinitionUpdate:
@@ -3020,13 +3013,6 @@ components:
           type: string
         dockerImageTag:
           type: string
-        normalizationRepository:
-          type: string
-        normalizationTag:
-          type: string
-        supportDbt:
-          type: boolean
-          default: false
         documentationUrl:
           type: string
           format: uri
diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java
index abf22541318d..afdf020993a3 100644
--- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java
+++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java
@@ -45,7 +45,6 @@ public static NormalizationRunner create(final String connectorImageName,
                                            final String normalizationImage) {
     final var valuePair = getNormalizationInfoForConnector(connectorImageName);
     final String factoryNormalizationImage = String.format("%s:%s", valuePair.getLeft(), normalizationVersion);
-    log.error("destination normalization runner factory -> {}", factoryNormalizationImage);
     if (Objects.nonNull(normalizationImage)
         && !normalizationImage.equalsIgnoreCase(factoryNormalizationImage)) {
       log.error(
diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java
index 417f21d792c5..0a995f4ba717 100644
--- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java
+++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java
@@ -95,9 +95,6 @@ static DestinationDefinitionRead buildDestinationDefinitionRead(final StandardDe
           .dockerRepository(standardDestinationDefinition.getDockerRepository())
           .dockerImageTag(standardDestinationDefinition.getDockerImageTag())
           .documentationUrl(new URI(standardDestinationDefinition.getDocumentationUrl()))
-          .normalizationRepository(standardDestinationDefinition.getNormalizationRepository())
-          .normalizationTag(standardDestinationDefinition.getNormalizationTag())
-          .supportDbt(standardDestinationDefinition.getSupportsDbt())
           .icon(loadIcon(standardDestinationDefinition.getIcon()))
           .protocolVersion(standardDestinationDefinition.getProtocolVersion())
           .releaseStage(getReleaseStage(standardDestinationDefinition))
@@ -234,9 +231,6 @@ private StandardDestinationDefinition destinationDefinitionFromCreate(final Dest
         .withSpec(spec)
         .withProtocolVersion(airbyteProtocolVersion.serialize())
         .withTombstone(false)
-        .withSupportsDbt(destinationDefCreate.getSupportDbt())
-        .withNormalizationRepository(destinationDefCreate.getNormalizationRepository())
-        .withNormalizationTag(destinationDefCreate.getNormalizationTag())
         .withReleaseStage(StandardDestinationDefinition.ReleaseStage.CUSTOM)
         .withResourceRequirements(ApiPojoConverters.actorDefResourceReqsToInternal(destinationDefCreate.getResourceRequirements()));
     return destinationDefinition;
diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java
index c63d4ed8e243..daad65516f51 100644
--- a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java
+++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java
@@ -111,9 +111,6 @@ private StandardDestinationDefinition generateDestinationDefinition() {
         .withTombstone(false)
         .withReleaseStage(StandardDestinationDefinition.ReleaseStage.ALPHA)
         .withReleaseDate(TODAY_DATE_STRING)
-        .withNormalizationTag("0.1.2")
-        .withNormalizationRepository("normalization-repo")
-        .withSupportsDbt(true)
         .withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(new ResourceRequirements().withCpuRequest("2")));
   }
 
@@ -131,9 +128,6 @@ void testListDestinations() throws JsonValidationException, IOException, URISynt
         .dockerImageTag(destinationDefinition.getDockerImageTag())
         .documentationUrl(new URI(destinationDefinition.getDocumentationUrl()))
         .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon()))
-        .normalizationRepository(destinationDefinition.getNormalizationRepository())
-        .normalizationTag(destinationDefinition.getNormalizationTag())
-        .supportDbt(destinationDefinition.getSupportsDbt())
         .protocolVersion(destinationDefinition.getProtocolVersion())
         .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value()))
         .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate()))
@@ -149,9 +143,6 @@ void testListDestinations() throws JsonValidationException, IOException, URISynt
         .dockerImageTag(destination2.getDockerImageTag())
         .documentationUrl(new URI(destination2.getDocumentationUrl()))
         .icon(DestinationDefinitionsHandler.loadIcon(destination2.getIcon()))
-        .normalizationRepository(destination2.getNormalizationRepository())
-        .normalizationTag(destination2.getNormalizationTag())
-        .supportDbt(destination2.getSupportsDbt())
         .protocolVersion(destinationDefinition.getProtocolVersion())
         .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value()))
         .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate()))
@@ -182,9 +173,6 @@ void testListDestinationDefinitionsForWorkspace() throws IOException, URISyntaxE
         .dockerImageTag(destinationDefinition.getDockerImageTag())
         .documentationUrl(new URI(destinationDefinition.getDocumentationUrl()))
         .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon()))
-        .normalizationRepository(destinationDefinition.getNormalizationRepository())
-        .normalizationTag(destinationDefinition.getNormalizationTag())
-        .supportDbt(destinationDefinition.getSupportsDbt())
         .protocolVersion(destinationDefinition.getProtocolVersion())
         .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value()))
         .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate()))
@@ -200,9 +188,6 @@ void testListDestinationDefinitionsForWorkspace() throws IOException, URISyntaxE
         .dockerImageTag(destination2.getDockerImageTag())
         .documentationUrl(new URI(destination2.getDocumentationUrl()))
         .icon(DestinationDefinitionsHandler.loadIcon(destination2.getIcon()))
-        .normalizationRepository(destination2.getNormalizationRepository())
-        .normalizationTag(destination2.getNormalizationTag())
-        .supportDbt(destination2.getSupportsDbt())
         .protocolVersion(destinationDefinition.getProtocolVersion())
         .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value()))
         .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate()))
@@ -236,9 +221,6 @@ void testListPrivateDestinationDefinitions() throws IOException, URISyntaxExcept
         .dockerImageTag(destinationDefinition.getDockerImageTag())
         .documentationUrl(new URI(destinationDefinition.getDocumentationUrl()))
         .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon()))
-        .normalizationRepository(destinationDefinition.getNormalizationRepository())
-        .normalizationTag(destinationDefinition.getNormalizationTag())
-        .supportDbt(destinationDefinition.getSupportsDbt())
         .protocolVersion(destinationDefinition.getProtocolVersion())
         .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value()))
         .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate()))
@@ -254,9 +236,6 @@ void testListPrivateDestinationDefinitions() throws IOException, URISyntaxExcept
         .dockerImageTag(destinationDefinition.getDockerImageTag())
         .documentationUrl(new URI(destinationDefinition.getDocumentationUrl()))
         .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon()))
-        .normalizationRepository(destinationDefinition.getNormalizationRepository())
-        .normalizationTag(destinationDefinition.getNormalizationTag())
-        .supportDbt(destinationDefinition.getSupportsDbt())
         .protocolVersion(destinationDefinition.getProtocolVersion())
         .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value()))
         .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate()))
@@ -293,9 +272,6 @@ void testGetDestination() throws JsonValidationException, ConfigNotFoundExceptio
         .dockerImageTag(destinationDefinition.getDockerImageTag())
         .documentationUrl(new URI(destinationDefinition.getDocumentationUrl()))
         .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon()))
-        .normalizationRepository(destinationDefinition.getNormalizationRepository())
-        .normalizationTag(destinationDefinition.getNormalizationTag())
-        .supportDbt(destinationDefinition.getSupportsDbt())
         .protocolVersion(destinationDefinition.getProtocolVersion())
         .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value()))
         .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate()))
@@ -342,9 +318,6 @@ void testGetDefinitionWithGrantForWorkspace() throws JsonValidationException, Co
         .dockerImageTag(destinationDefinition.getDockerImageTag())
         .documentationUrl(new URI(destinationDefinition.getDocumentationUrl()))
         .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon()))
-        .normalizationRepository(destinationDefinition.getNormalizationRepository())
-        .normalizationTag(destinationDefinition.getNormalizationTag())
-        .supportDbt(destinationDefinition.getSupportsDbt())
         .protocolVersion(destinationDefinition.getProtocolVersion())
         .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value()))
         .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate()))
@@ -380,9 +353,6 @@ void testCreateDestinationDefinition() throws URISyntaxException, IOException, J
         .dockerImageTag(destination.getDockerImageTag())
         .documentationUrl(new URI(destination.getDocumentationUrl()))
         .icon(destination.getIcon())
-        .normalizationRepository(destination.getNormalizationRepository())
-        .normalizationTag(destination.getNormalizationTag())
-        .supportDbt(destination.getSupportsDbt())
         .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements()
             ._default(new io.airbyte.api.model.generated.ResourceRequirements()
                 .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest()))
@@ -395,9 +365,6 @@ void testCreateDestinationDefinition() throws URISyntaxException, IOException, J
         .documentationUrl(new URI(destination.getDocumentationUrl()))
         .destinationDefinitionId(destination.getDestinationDefinitionId())
         .icon(DestinationDefinitionsHandler.loadIcon(destination.getIcon()))
-        .normalizationRepository(destination.getNormalizationRepository())
-        .normalizationTag(destination.getNormalizationTag())
-        .supportDbt(destination.getSupportsDbt())
         .protocolVersion(DEFAULT_PROTOCOL_VERSION)
         .releaseStage(ReleaseStage.CUSTOM)
         .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements()
@@ -465,9 +432,6 @@ void testCreateCustomDestinationDefinition() throws URISyntaxException, IOExcept
         .dockerImageTag(destination.getDockerImageTag())
         .documentationUrl(new URI(destination.getDocumentationUrl()))
         .icon(destination.getIcon())
-        .normalizationRepository(destination.getNormalizationRepository())
-        .normalizationTag(destination.getNormalizationTag())
-        .supportDbt(destination.getSupportsDbt())
         .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements()
             ._default(new io.airbyte.api.model.generated.ResourceRequirements()
                 .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest()))
@@ -484,9 +448,6 @@ void testCreateCustomDestinationDefinition() throws URISyntaxException, IOExcept
         .documentationUrl(new URI(destination.getDocumentationUrl()))
         .destinationDefinitionId(destination.getDestinationDefinitionId())
         .icon(DestinationDefinitionsHandler.loadIcon(destination.getIcon()))
-        .normalizationRepository(destination.getNormalizationRepository())
-        .normalizationTag(destination.getNormalizationTag())
-        .supportDbt(destination.getSupportsDbt())
         .protocolVersion(DEFAULT_PROTOCOL_VERSION)
         .releaseStage(ReleaseStage.CUSTOM)
         .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements()
@@ -652,9 +613,6 @@ void testGrantDestinationDefinitionToWorkspace() throws JsonValidationException,
         .dockerImageTag(destinationDefinition.getDockerImageTag())
         .documentationUrl(new URI(destinationDefinition.getDocumentationUrl()))
         .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon()))
-        .normalizationRepository(destinationDefinition.getNormalizationRepository())
-        .normalizationTag(destinationDefinition.getNormalizationTag())
-        .supportDbt(destinationDefinition.getSupportsDbt())
         .protocolVersion(destinationDefinition.getProtocolVersion())
         .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value()))
         .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate()))
diff --git a/airbyte-worker-models/src/main/resources/workers_models/IntegrationLauncherConfig.yaml b/airbyte-worker-models/src/main/resources/workers_models/IntegrationLauncherConfig.yaml
index ce339b01fcb2..d3c24d7aa12f 100644
--- a/airbyte-worker-models/src/main/resources/workers_models/IntegrationLauncherConfig.yaml
+++ b/airbyte-worker-models/src/main/resources/workers_models/IntegrationLauncherConfig.yaml
@@ -18,7 +18,7 @@ properties:
     type: string
   normalizationDockerImage:
     type: string
-  supportDbt:
+  supportsDbt:
     type: boolean
     default: false
   protocolVersion:
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java
index db1a99fdf0cd..8081cedd4dfc 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java
@@ -109,7 +109,7 @@ public GeneratedJobInput getSyncWorkflowInput(final SyncInput input) {
           .withDockerImage(config.getDestinationDockerImage())
           .withProtocolVersion(config.getDestinationProtocolVersion())
           .withNormalizationDockerImage(destinationNormalizationDockerImage)
-          .withSupportDbt(supportDbt);
+          .withSupportsDbt(supportDbt);
 
       final StandardSyncInput syncInput = new StandardSyncInput()
           .withNamespaceDefinition(config.getNamespaceDefinition())
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java
index 88636e539fef..89ca22f61eed 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java
@@ -47,9 +47,7 @@
 import java.util.Optional;
 import java.util.UUID;
 import java.util.function.Supplier;
-import lombok.extern.slf4j.Slf4j;
 
-@Slf4j
 @Singleton
 public class NormalizationActivityImpl implements NormalizationActivity {
 
@@ -149,8 +147,6 @@ public NormalizationInput generateNormalizationInput(final StandardSyncInput syn
   private CheckedSupplier, Exception> getLegacyWorkerFactory(
                                                                                                               final IntegrationLauncherConfig destinationLauncherConfig,
                                                                                                               final JobRunConfig jobRunConfig) {
-    log.error("destination DBT -> {}", destinationLauncherConfig.getSupportDbt());
-    log.error("destination normalization -> {}", destinationLauncherConfig.getNormalizationDockerImage());
     return () -> new DefaultNormalizationWorker(
         jobRunConfig.getJobId(),
         Math.toIntExact(jobRunConfig.getAttemptId()),

From f1cf84dc0d90c81b4e304e19782d857ffa164299 Mon Sep 17 00:00:00 2001
From: Andrii Korotkov 
Date: Wed, 30 Nov 2022 10:11:19 +0200
Subject: [PATCH 09/34] fixed remarks

---
 .../api/generated-api-html/index.html         | 214 +++++++-----------
 1 file changed, 80 insertions(+), 134 deletions(-)

diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html
index 347c1eded3c7..b1b26c98dd65 100644
--- a/docs/reference/api/generated-api-html/index.html
+++ b/docs/reference/api/generated-api-html/index.html
@@ -2617,12 +2617,6 @@ 
Return type

     
Example data

     
Content-Type: application/json

     
{
-  "documentationUrl" : "https://openapi-generator.tech",
-  "releaseDate" : "2000-01-23",
-  "dockerRepository" : "dockerRepository",
-  "normalizationRepository" : "normalizationRepository",
-  "icon" : "icon",
-  "supportDbt" : false,
   "resourceRequirements" : {
     "default" : {
       "cpu_limit" : "cpu_limit",
@@ -2646,11 +2640,14 @@ 
Example data

       }
     } ]
   },
+  "documentationUrl" : "https://openapi-generator.tech",
   "dockerImageTag" : "dockerImageTag",
+  "releaseDate" : "2000-01-23",
+  "dockerRepository" : "dockerRepository",
   "name" : "name",
+  "icon" : "icon",
   "protocolVersion" : "protocolVersion",
-  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "normalizationTag" : "normalizationTag"
+  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
 }

 
     
Produces

@@ -2705,12 +2702,6 @@ 
Return type

     
Example data

     
Content-Type: application/json

     
{
-  "documentationUrl" : "https://openapi-generator.tech",
-  "releaseDate" : "2000-01-23",
-  "dockerRepository" : "dockerRepository",
-  "normalizationRepository" : "normalizationRepository",
-  "icon" : "icon",
-  "supportDbt" : false,
   "resourceRequirements" : {
     "default" : {
       "cpu_limit" : "cpu_limit",
@@ -2734,11 +2725,14 @@ 
Example data

       }
     } ]
   },
+  "documentationUrl" : "https://openapi-generator.tech",
   "dockerImageTag" : "dockerImageTag",
+  "releaseDate" : "2000-01-23",
+  "dockerRepository" : "dockerRepository",
   "name" : "name",
+  "icon" : "icon",
   "protocolVersion" : "protocolVersion",
-  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "normalizationTag" : "normalizationTag"
+  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
 }

 
     
Produces

@@ -2889,12 +2883,6 @@ 
Return type

     
Example data

     
Content-Type: application/json

     
{
-  "documentationUrl" : "https://openapi-generator.tech",
-  "releaseDate" : "2000-01-23",
-  "dockerRepository" : "dockerRepository",
-  "normalizationRepository" : "normalizationRepository",
-  "icon" : "icon",
-  "supportDbt" : false,
   "resourceRequirements" : {
     "default" : {
       "cpu_limit" : "cpu_limit",
@@ -2918,11 +2906,14 @@ 
Example data

       }
     } ]
   },
+  "documentationUrl" : "https://openapi-generator.tech",
   "dockerImageTag" : "dockerImageTag",
+  "releaseDate" : "2000-01-23",
+  "dockerRepository" : "dockerRepository",
   "name" : "name",
+  "icon" : "icon",
   "protocolVersion" : "protocolVersion",
-  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "normalizationTag" : "normalizationTag"
+  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
 }

 
     
Produces

@@ -2980,12 +2971,6 @@ 
Return type

     
Example data

     
Content-Type: application/json

     
{
-  "documentationUrl" : "https://openapi-generator.tech",
-  "releaseDate" : "2000-01-23",
-  "dockerRepository" : "dockerRepository",
-  "normalizationRepository" : "normalizationRepository",
-  "icon" : "icon",
-  "supportDbt" : false,
   "resourceRequirements" : {
     "default" : {
       "cpu_limit" : "cpu_limit",
@@ -3009,11 +2994,14 @@ 
Example data

       }
     } ]
   },
+  "documentationUrl" : "https://openapi-generator.tech",
   "dockerImageTag" : "dockerImageTag",
+  "releaseDate" : "2000-01-23",
+  "dockerRepository" : "dockerRepository",
   "name" : "name",
+  "icon" : "icon",
   "protocolVersion" : "protocolVersion",
-  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "normalizationTag" : "normalizationTag"
+  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
 }

 
     
Produces

@@ -3072,12 +3060,6 @@ 
Example data

     
Content-Type: application/json

     
{
   "destinationDefinition" : {
-    "documentationUrl" : "https://openapi-generator.tech",
-    "releaseDate" : "2000-01-23",
-    "dockerRepository" : "dockerRepository",
-    "normalizationRepository" : "normalizationRepository",
-    "icon" : "icon",
-    "supportDbt" : false,
     "resourceRequirements" : {
       "default" : {
         "cpu_limit" : "cpu_limit",
@@ -3101,11 +3083,14 @@ 
Example data

         }
       } ]
     },
+    "documentationUrl" : "https://openapi-generator.tech",
     "dockerImageTag" : "dockerImageTag",
+    "releaseDate" : "2000-01-23",
+    "dockerRepository" : "dockerRepository",
     "name" : "name",
+    "icon" : "icon",
     "protocolVersion" : "protocolVersion",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "normalizationTag" : "normalizationTag"
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   },
   "granted" : true
 }

@@ -3154,12 +3139,6 @@ 
Example data

     
Content-Type: application/json

     
{
   "destinationDefinitions" : [ {
-    "documentationUrl" : "https://openapi-generator.tech",
-    "releaseDate" : "2000-01-23",
-    "dockerRepository" : "dockerRepository",
-    "normalizationRepository" : "normalizationRepository",
-    "icon" : "icon",
-    "supportDbt" : false,
     "resourceRequirements" : {
       "default" : {
         "cpu_limit" : "cpu_limit",
@@ -3183,18 +3162,15 @@ 
Example data

         }
       } ]
     },
-    "dockerImageTag" : "dockerImageTag",
-    "name" : "name",
-    "protocolVersion" : "protocolVersion",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "normalizationTag" : "normalizationTag"
-  }, {
     "documentationUrl" : "https://openapi-generator.tech",
+    "dockerImageTag" : "dockerImageTag",
     "releaseDate" : "2000-01-23",
     "dockerRepository" : "dockerRepository",
-    "normalizationRepository" : "normalizationRepository",
+    "name" : "name",
     "icon" : "icon",
-    "supportDbt" : false,
+    "protocolVersion" : "protocolVersion",
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  }, {
     "resourceRequirements" : {
       "default" : {
         "cpu_limit" : "cpu_limit",
@@ -3218,11 +3194,14 @@ 
Example data

         }
       } ]
     },
+    "documentationUrl" : "https://openapi-generator.tech",
     "dockerImageTag" : "dockerImageTag",
+    "releaseDate" : "2000-01-23",
+    "dockerRepository" : "dockerRepository",
     "name" : "name",
+    "icon" : "icon",
     "protocolVersion" : "protocolVersion",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "normalizationTag" : "normalizationTag"
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   } ]
 }

 
@@ -3276,12 +3255,6 @@ 
Example data

     
Content-Type: application/json

     
{
   "destinationDefinitions" : [ {
-    "documentationUrl" : "https://openapi-generator.tech",
-    "releaseDate" : "2000-01-23",
-    "dockerRepository" : "dockerRepository",
-    "normalizationRepository" : "normalizationRepository",
-    "icon" : "icon",
-    "supportDbt" : false,
     "resourceRequirements" : {
       "default" : {
         "cpu_limit" : "cpu_limit",
@@ -3305,18 +3278,15 @@ 
Example data

         }
       } ]
     },
-    "dockerImageTag" : "dockerImageTag",
-    "name" : "name",
-    "protocolVersion" : "protocolVersion",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "normalizationTag" : "normalizationTag"
-  }, {
     "documentationUrl" : "https://openapi-generator.tech",
+    "dockerImageTag" : "dockerImageTag",
     "releaseDate" : "2000-01-23",
     "dockerRepository" : "dockerRepository",
-    "normalizationRepository" : "normalizationRepository",
+    "name" : "name",
     "icon" : "icon",
-    "supportDbt" : false,
+    "protocolVersion" : "protocolVersion",
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  }, {
     "resourceRequirements" : {
       "default" : {
         "cpu_limit" : "cpu_limit",
@@ -3340,11 +3310,14 @@ 
Example data

         }
       } ]
     },
+    "documentationUrl" : "https://openapi-generator.tech",
     "dockerImageTag" : "dockerImageTag",
+    "releaseDate" : "2000-01-23",
+    "dockerRepository" : "dockerRepository",
     "name" : "name",
+    "icon" : "icon",
     "protocolVersion" : "protocolVersion",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "normalizationTag" : "normalizationTag"
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   } ]
 }

 
@@ -3386,12 +3359,6 @@ 
Example data

     
Content-Type: application/json

     
{
   "destinationDefinitions" : [ {
-    "documentationUrl" : "https://openapi-generator.tech",
-    "releaseDate" : "2000-01-23",
-    "dockerRepository" : "dockerRepository",
-    "normalizationRepository" : "normalizationRepository",
-    "icon" : "icon",
-    "supportDbt" : false,
     "resourceRequirements" : {
       "default" : {
         "cpu_limit" : "cpu_limit",
@@ -3415,18 +3382,15 @@ 
Example data

         }
       } ]
     },
-    "dockerImageTag" : "dockerImageTag",
-    "name" : "name",
-    "protocolVersion" : "protocolVersion",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "normalizationTag" : "normalizationTag"
-  }, {
     "documentationUrl" : "https://openapi-generator.tech",
+    "dockerImageTag" : "dockerImageTag",
     "releaseDate" : "2000-01-23",
     "dockerRepository" : "dockerRepository",
-    "normalizationRepository" : "normalizationRepository",
+    "name" : "name",
     "icon" : "icon",
-    "supportDbt" : false,
+    "protocolVersion" : "protocolVersion",
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  }, {
     "resourceRequirements" : {
       "default" : {
         "cpu_limit" : "cpu_limit",
@@ -3450,11 +3414,14 @@ 
Example data

         }
       } ]
     },
+    "documentationUrl" : "https://openapi-generator.tech",
     "dockerImageTag" : "dockerImageTag",
+    "releaseDate" : "2000-01-23",
+    "dockerRepository" : "dockerRepository",
     "name" : "name",
+    "icon" : "icon",
     "protocolVersion" : "protocolVersion",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "normalizationTag" : "normalizationTag"
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   } ]
 }

 
@@ -3509,12 +3476,6 @@ 
Example data

     
{
   "destinationDefinitions" : [ {
     "destinationDefinition" : {
-      "documentationUrl" : "https://openapi-generator.tech",
-      "releaseDate" : "2000-01-23",
-      "dockerRepository" : "dockerRepository",
-      "normalizationRepository" : "normalizationRepository",
-      "icon" : "icon",
-      "supportDbt" : false,
       "resourceRequirements" : {
         "default" : {
           "cpu_limit" : "cpu_limit",
@@ -3538,21 +3499,18 @@ 
Example data

           }
         } ]
       },
+      "documentationUrl" : "https://openapi-generator.tech",
       "dockerImageTag" : "dockerImageTag",
+      "releaseDate" : "2000-01-23",
+      "dockerRepository" : "dockerRepository",
       "name" : "name",
+      "icon" : "icon",
       "protocolVersion" : "protocolVersion",
-      "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-      "normalizationTag" : "normalizationTag"
+      "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
     },
     "granted" : true
   }, {
     "destinationDefinition" : {
-      "documentationUrl" : "https://openapi-generator.tech",
-      "releaseDate" : "2000-01-23",
-      "dockerRepository" : "dockerRepository",
-      "normalizationRepository" : "normalizationRepository",
-      "icon" : "icon",
-      "supportDbt" : false,
       "resourceRequirements" : {
         "default" : {
           "cpu_limit" : "cpu_limit",
@@ -3576,11 +3534,14 @@ 
Example data

           }
         } ]
       },
+      "documentationUrl" : "https://openapi-generator.tech",
       "dockerImageTag" : "dockerImageTag",
+      "releaseDate" : "2000-01-23",
+      "dockerRepository" : "dockerRepository",
       "name" : "name",
+      "icon" : "icon",
       "protocolVersion" : "protocolVersion",
-      "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-      "normalizationTag" : "normalizationTag"
+      "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
     },
     "granted" : true
   } ]
@@ -3683,12 +3644,6 @@ 
Return type

     
Example data

     
Content-Type: application/json

     
{
-  "documentationUrl" : "https://openapi-generator.tech",
-  "releaseDate" : "2000-01-23",
-  "dockerRepository" : "dockerRepository",
-  "normalizationRepository" : "normalizationRepository",
-  "icon" : "icon",
-  "supportDbt" : false,
   "resourceRequirements" : {
     "default" : {
       "cpu_limit" : "cpu_limit",
@@ -3712,11 +3667,14 @@ 
Example data

       }
     } ]
   },
+  "documentationUrl" : "https://openapi-generator.tech",
   "dockerImageTag" : "dockerImageTag",
+  "releaseDate" : "2000-01-23",
+  "dockerRepository" : "dockerRepository",
   "name" : "name",
+  "icon" : "icon",
   "protocolVersion" : "protocolVersion",
-  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "normalizationTag" : "normalizationTag"
+  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
 }

 
     
Produces

@@ -3774,12 +3732,6 @@ 
Return type

     
Example data

     
Content-Type: application/json

     
{
-  "documentationUrl" : "https://openapi-generator.tech",
-  "releaseDate" : "2000-01-23",
-  "dockerRepository" : "dockerRepository",
-  "normalizationRepository" : "normalizationRepository",
-  "icon" : "icon",
-  "supportDbt" : false,
   "resourceRequirements" : {
     "default" : {
       "cpu_limit" : "cpu_limit",
@@ -3803,11 +3755,14 @@ 
Example data

       }
     } ]
   },
+  "documentationUrl" : "https://openapi-generator.tech",
   "dockerImageTag" : "dockerImageTag",
+  "releaseDate" : "2000-01-23",
+  "dockerRepository" : "dockerRepository",
   "name" : "name",
+  "icon" : "icon",
   "protocolVersion" : "protocolVersion",
-  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "normalizationTag" : "normalizationTag"
+  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
 }

 
     
Produces

@@ -4699,12 +4654,6 @@ 
Example data

     "airbyteVersion" : "airbyteVersion",
     "id" : 0,
     "destinationDefinition" : {
-      "documentationUrl" : "https://openapi-generator.tech",
-      "releaseDate" : "2000-01-23",
-      "dockerRepository" : "dockerRepository",
-      "normalizationRepository" : "normalizationRepository",
-      "icon" : "icon",
-      "supportDbt" : false,
       "resourceRequirements" : {
         "default" : {
           "cpu_limit" : "cpu_limit",
@@ -4728,11 +4677,14 @@ 
Example data

           }
         } ]
       },
+      "documentationUrl" : "https://openapi-generator.tech",
       "dockerImageTag" : "dockerImageTag",
+      "releaseDate" : "2000-01-23",
+      "dockerRepository" : "dockerRepository",
       "name" : "name",
+      "icon" : "icon",
       "protocolVersion" : "protocolVersion",
-      "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-      "normalizationTag" : "normalizationTag"
+      "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
     }
   },
   "attempts" : [ {
@@ -11156,9 +11108,6 @@ 
DestinationDefinitionCreatedockerImageTag 
String  

 
documentationUrl 
URI  format: uri

 
icon (optional)
String  

-
normalizationRepository (optional)
String  

-
normalizationTag (optional)
String  

-
supportDbt (optional)
Boolean  

 
resourceRequirements (optional)
ActorDefinitionResourceRequirements  

       
   
@@ -11185,9 +11134,6 @@ 
DestinationDefinitionRead -
 
name 
String  

 
dockerRepository 
String  

 
dockerImageTag 
String  

-
normalizationRepository (optional)
String  

-
normalizationTag (optional)
String  

-
supportDbt (optional)
Boolean  

 
documentationUrl 
URI  format: uri

 
icon (optional)
String  

 
protocolVersion (optional)
String The Airbyte Protocol version supported by the connector 


From d972133c38f1193d4e444cf1edd5e9d5c4e9e98b Mon Sep 17 00:00:00 2001
From: Andrii Korotkov 
Date: Thu, 1 Dec 2022 17:51:58 +0200
Subject: [PATCH 10/34] fixed remarks

---
 .../DefaultNormalizationRunner.java              | 16 ----------------
 .../types/StandardDestinationDefinition.yaml     |  1 -
 2 files changed, 17 deletions(-)

diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
index 8c960aaae1fb..44b46dd5bc3d 100644
--- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
+++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
@@ -54,22 +54,6 @@ public class DefaultNormalizationRunner implements NormalizationRunner {
 
   private Process process = null;
 
-  public DefaultNormalizationRunner(final ProcessFactory processFactory,
-                                    final String normalizationImage) {
-    this.processFactory = processFactory;
-    this.normalizationImageName = normalizationImage;
-    this.destinationType = getDestinationTypeFromNormalizationImageName(normalizationImage);
-  }
-
-  private String getDestinationTypeFromNormalizationImageName(final String normalizationImage) {
-    final String imageNameWithoutTag = normalizationImage.contains(":") ? normalizationImage.split(":")[0] : normalizationImage;
-    if ("airbyte/normalization".equalsIgnoreCase(imageNameWithoutTag)) {
-      return "normalization";
-    } else {
-      return imageNameWithoutTag.replace("airbyte/normalization-", "");
-    }
-  }
-
   public DefaultNormalizationRunner(final ProcessFactory processFactory,
                                     final String normalizationImage) {
     this.processFactory = processFactory;
diff --git a/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml b/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml
index f3f37ebb3971..4c51a6b3239c 100644
--- a/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml
+++ b/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml
@@ -64,4 +64,3 @@ properties:
     type: string
   supportsDbt:
     type: boolean
-    default: false

From f97da1de018b066ac0b6151d9d04550ee3a1ffb4 Mon Sep 17 00:00:00 2001
From: Andrii Korotkov 
Date: Thu, 1 Dec 2022 19:29:24 +0200
Subject: [PATCH 11/34] updated acceptance tests

---
 .../bases/standard-destination-test/build.gradle  |  1 +
 .../destination/DestinationAcceptanceTest.java    | 15 +++++++++++++--
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/airbyte-integrations/bases/standard-destination-test/build.gradle b/airbyte-integrations/bases/standard-destination-test/build.gradle
index 48568f05ace5..7b73aa3d1da7 100644
--- a/airbyte-integrations/bases/standard-destination-test/build.gradle
+++ b/airbyte-integrations/bases/standard-destination-test/build.gradle
@@ -6,6 +6,7 @@ dependencies {
     implementation project(':airbyte-commons-worker')
     implementation project(':airbyte-config:config-models')
     implementation project(':airbyte-config:config-persistence')
+    implementation project(':airbyte-config:init')
     implementation project(':airbyte-json-validation')
     implementation project(':airbyte-integrations:bases:base-java')
     implementation project(':airbyte-protocol:protocol-models')
diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
index d43f593bfab3..147b73470fcb 100644
--- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
+++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
@@ -30,7 +30,9 @@
 import io.airbyte.config.StandardCheckConnectionInput;
 import io.airbyte.config.StandardCheckConnectionOutput;
 import io.airbyte.config.StandardCheckConnectionOutput.Status;
+import io.airbyte.config.StandardDestinationDefinition;
 import io.airbyte.config.WorkerDestinationConfig;
+import io.airbyte.config.init.LocalDefinitionsProvider;
 import io.airbyte.integrations.destination.NamingConventionTransformer;
 import io.airbyte.integrations.standardtest.destination.argproviders.DataArgumentsProvider;
 import io.airbyte.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider;
@@ -96,7 +98,6 @@
 public abstract class DestinationAcceptanceTest {
 
   private static final Random RANDOM = new Random();
-  private static final String NORMALIZATION_VERSION = "dev";
 
   private static final String JOB_ID = "0";
   private static final int JOB_ATTEMPT = 0;
@@ -122,7 +123,17 @@ public abstract class DestinationAcceptanceTest {
   protected abstract String getImageName();
 
   protected String getNormalizationImageName() {
-    return null;
+    try {
+      LocalDefinitionsProvider provider = new LocalDefinitionsProvider(LocalDefinitionsProvider.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS);
+      Optional optionalDefinition = provider.getDestinationDefinitions().stream()
+          .filter(definition -> getImageName().equalsIgnoreCase(definition.getDockerRepository()))
+          .findFirst();
+      return optionalDefinition
+          .map(standardDestinationDefinition -> standardDestinationDefinition.getNormalizationRepository() + ":" + standardDestinationDefinition.getDockerImageTag())
+          .orElse(null);
+    } catch (IOException e) {
+      return null;
+    }
   }
 
   /**

From ee82e5487367156eecf8615280828702abf11946 Mon Sep 17 00:00:00 2001
From: Andrii Korotkov 
Date: Fri, 2 Dec 2022 14:58:38 +0200
Subject: [PATCH 12/34] updated acceptance tests

---
 .../DefaultNormalizationRunner.java           |  8 ++-
 .../DestinationAcceptanceTest.java            | 69 ++++++++++---------
 2 files changed, 42 insertions(+), 35 deletions(-)

diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
index 44b46dd5bc3d..4027d9135a15 100644
--- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
+++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
@@ -58,7 +58,13 @@ public DefaultNormalizationRunner(final ProcessFactory processFactory,
                                     final String normalizationImage) {
     this.processFactory = processFactory;
     this.normalizationImageName = normalizationImage;
-    this.destinationType = null;
+    this.destinationType = getDestinationTypeFromNormalizationImage(normalizationImage);
+  }
+
+  private String getDestinationTypeFromNormalizationImage(final String normalizationImage) {
+    String imageNameWithoutTag = normalizationImage.contains(":") ? normalizationImage.split(":")[0] : normalizationImage;
+    return "airbyte/normalization".equalsIgnoreCase(imageNameWithoutTag) ? "normalization"
+        : imageNameWithoutTag.replace("airbyte/normalization-", "");
   }
 
   @Override
diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
index 147b73470fcb..308007e1b3e7 100644
--- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
+++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
@@ -74,6 +74,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Random;
 import java.util.UUID;
@@ -122,20 +123,28 @@ public abstract class DestinationAcceptanceTest {
    */
   protected abstract String getImageName();
 
-  protected String getNormalizationImageName() {
+  private String getImageNameWithoutTag() {
+    return getImageName().contains(":") ? getImageName().split(":")[0] : getImageName();
+  }
+
+  private Optional getOptionalDestinationDefinitionFromProvider(final String imageNameWithoutTag) {
     try {
       LocalDefinitionsProvider provider = new LocalDefinitionsProvider(LocalDefinitionsProvider.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS);
-      Optional optionalDefinition = provider.getDestinationDefinitions().stream()
-          .filter(definition -> getImageName().equalsIgnoreCase(definition.getDockerRepository()))
-          .findFirst();
-      return optionalDefinition
-          .map(standardDestinationDefinition -> standardDestinationDefinition.getNormalizationRepository() + ":" + standardDestinationDefinition.getDockerImageTag())
-          .orElse(null);
+      return provider.getDestinationDefinitions().stream()
+              .filter(definition -> imageNameWithoutTag.equalsIgnoreCase(definition.getDockerRepository()))
+              .findFirst();
     } catch (IOException e) {
-      return null;
+      return Optional.empty();
     }
   }
 
+  protected String getNormalizationImageName() {
+    return getOptionalDestinationDefinitionFromProvider(getImageNameWithoutTag())
+      .map(standardDestinationDefinition -> standardDestinationDefinition.getNormalizationRepository() + ":"
+          + standardDestinationDefinition.getDockerImageTag())
+      .orElse(null);
+  }
+
   /**
    * Configuration specific to the integration. Will be passed to integration where appropriate in
    * each test. Should be valid.
@@ -215,24 +224,16 @@ protected boolean implementsAppend() throws WorkerException {
     }
   }
 
-  protected boolean normalizationFromSpec() throws Exception {
-    final ConnectorSpecification spec = runSpec();
-    assertNotNull(spec);
-    if (spec.getSupportsNormalization() != null) {
-      return spec.getSupportsNormalization();
-    } else {
-      return false;
-    }
+  protected boolean normalizationFromDefinition() {
+    return getOptionalDestinationDefinitionFromProvider(getImageNameWithoutTag())
+        .map(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getNormalizationRepository()) && Objects.nonNull(standardDestinationDefinition.getNormalizationTag()))
+        .orElse(false);
   }
 
-  protected boolean dbtFromSpec() throws WorkerException {
-    final ConnectorSpecification spec = runSpec();
-    assertNotNull(spec);
-    if (spec.getSupportsDBT() != null) {
-      return spec.getSupportsDBT();
-    } else {
-      return false;
-    }
+  protected boolean dbtFromDefinition() {
+    return getOptionalDestinationDefinitionFromProvider(getImageNameWithoutTag())
+        .map(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getSupportsDbt()) && standardDestinationDefinition.getSupportsDbt())
+        .orElse(false);
   }
 
   /**
@@ -293,7 +294,7 @@ protected boolean implementsRecordSizeLimitChecks() {
    * Same idea as {@link #retrieveRecords(TestDestinationEnv, String, String, JsonNode)}. Except this
    * method should pull records from the table that contains the normalized records and convert them
    * back into the data as it would appear in an {@link AirbyteRecordMessage}. Only need to override
-   * this method if {@link #normalizationFromSpec} returns true.
+   * this method if {@link #normalizationFromDefinition} returns true.
    *
    * @param testEnv - information about the test environment.
    * @param streamName - name of the stream for which we are retrieving records.
@@ -552,9 +553,9 @@ public void testLineBreakCharacters() throws Exception {
 
   @Test
   public void specNormalizationValueShouldBeCorrect() throws Exception {
-    final boolean normalizationFromSpec = normalizationFromSpec();
-    assertEquals(normalizationFromSpec, supportsNormalization());
-    if (normalizationFromSpec) {
+    final boolean normalizationFromDefinition = normalizationFromDefinition();
+    assertEquals(normalizationFromDefinition, supportsNormalization());
+    if (normalizationFromDefinition) {
       boolean normalizationRunnerFactorySupportsDestinationImage;
       try {
         new DefaultNormalizationRunner(
@@ -564,13 +565,13 @@ public void specNormalizationValueShouldBeCorrect() throws Exception {
       } catch (final IllegalStateException e) {
         normalizationRunnerFactorySupportsDestinationImage = false;
       }
-      assertEquals(normalizationFromSpec, normalizationRunnerFactorySupportsDestinationImage);
+      assertEquals(normalizationFromDefinition, normalizationRunnerFactorySupportsDestinationImage);
     }
   }
 
   @Test
-  public void specDBTValueShouldBeCorrect() throws WorkerException {
-    assertEquals(dbtFromSpec(), supportsDBT());
+  public void specDBTValueShouldBeCorrect() {
+    assertEquals(dbtFromDefinition(), supportsDBT());
   }
 
   /**
@@ -639,7 +640,7 @@ public void testIncrementalSync() throws Exception {
   @ArgumentsSource(DataArgumentsProvider.class)
   public void testSyncWithNormalization(final String messagesFilename, final String catalogFilename)
       throws Exception {
-    if (!normalizationFromSpec()) {
+    if (!normalizationFromDefinition()) {
       return;
     }
 
@@ -846,7 +847,7 @@ protected int getMaxRecordValueLimit() {
 
   @Test
   public void testCustomDbtTransformations() throws Exception {
-    if (!dbtFromSpec()) {
+    if (!dbtFromDefinition()) {
       return;
     }
 
@@ -927,7 +928,7 @@ public void testCustomDbtTransformations() throws Exception {
 
   @Test
   void testCustomDbtTransformationsFailure() throws Exception {
-    if (!normalizationFromSpec() || !dbtFromSpec()) {
+    if (!normalizationFromDefinition() || !dbtFromDefinition()) {
       // we require normalization implementation for this destination, because we make sure to install
       // required dbt dependency in the normalization docker image in order to run this test successfully
       // (we don't actually rely on normalization running anything here though)

From 77fe4367683391da88f8ab4c170c73ea6f0f1f15 Mon Sep 17 00:00:00 2001
From: Andrii Korotkov 
Date: Sun, 4 Dec 2022 19:18:09 +0200
Subject: [PATCH 13/34] updated check_images_exist.sh script

---
 tools/bin/check_images_exist.sh | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/tools/bin/check_images_exist.sh b/tools/bin/check_images_exist.sh
index 1ffbf8195d0c..2098a9c73218 100755
--- a/tools/bin/check_images_exist.sh
+++ b/tools/bin/check_images_exist.sh
@@ -82,16 +82,18 @@ checkPlatformImages() {
 
 checkNormalizationImages() {
   echo -e "$blue_text""Checking Normalization images exist...""$default_text"
-  # the only way to know what version of normalization the platform is using is looking in NormalizationRunnerFactory.
   local image_version;
-  factory_path=airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java
+  definition_file_path=airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
   # -f True if file exists and is a regular file
-  if ! test -f $factory_path; then
-    echo -e "$red_text""No NormalizationRunnerFactory found at path! H4LP!!!""$default_text"
+  if ! test -f $definition_file_path; then
+    echo -e "$red_text""Destination definition file not found at path! H4LP!!!""$default_text"
   fi
-  image_version=$(cat $factory_path | grep 'NORMALIZATION_VERSION =' | cut -d"=" -f2 | sed 's:;::' | sed -e 's:"::g' | sed -e 's:[[:space:]]::g')
-  echo -e "$blue_text""Checking normalization images with version $image_version exist...""$default_text"
-  VERSION=$image_version 
+  normalization_image_versions=$(cat $definition_file_path | grep 'normalizationTag:' | cut -d":" -f2 | sed 's:;::' | sed -e 's:"::g' | sed -e 's:[[:space:]]::g')
+  IFS=' ' read -r -a array <<< "$normalization_image_versions"
+  # Get the first value of the normalization tag
+  normalization_image=${array[0]}
+  echo -e "$blue_text""Checking normalization images with version $normalization_image exist...""$default_text"
+  VERSION=$normalization_image
   check_compose_image_exist airbyte-integrations/bases/base-normalization/docker-compose.yaml $VERSION
 }
 

From 3afde21aa98c8c61f97f5d7ef43c040596559a3a Mon Sep 17 00:00:00 2001
From: Andrii Korotkov 
Date: Sun, 4 Dec 2022 19:56:50 +0200
Subject: [PATCH 14/34] updated method for get normalization image name for
 destination acceptance test

---
 .../standardtest/destination/DestinationAcceptanceTest.java   | 2 +-
 .../destination/mysql/MySQLDestinationAcceptanceTest.java     | 4 ----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
index 308007e1b3e7..b494b2a9bc4e 100644
--- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
+++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
@@ -141,7 +141,7 @@ private Optional getOptionalDestinationDefinition
   protected String getNormalizationImageName() {
     return getOptionalDestinationDefinitionFromProvider(getImageNameWithoutTag())
       .map(standardDestinationDefinition -> standardDestinationDefinition.getNormalizationRepository() + ":"
-          + standardDestinationDefinition.getDockerImageTag())
+          + standardDestinationDefinition.getNormalizationTag())
       .orElse(null);
   }
 
diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java
index 13fc7e2bad5c..0d88daceab55 100644
--- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java
+++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java
@@ -50,10 +50,6 @@ protected String getImageName() {
     return "airbyte/destination-mysql:dev";
   }
 
-  @Override
-  protected String getNormalizationImageName() {
-    return "airbyte/normalization-mysql:0.2.24";
-  }
 
   @Override
   protected boolean supportsDBT() {

From f91c1b1c258aa616a441d8aac0b3928103ed72ac Mon Sep 17 00:00:00 2001
From: Andrii Korotkov 
Date: Sun, 4 Dec 2022 20:49:36 +0200
Subject: [PATCH 15/34] fixed code style

---
 .../destination/DestinationAcceptanceTest.java   | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
index b494b2a9bc4e..2d701a37d1e0 100644
--- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
+++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
@@ -131,8 +131,8 @@ private Optional getOptionalDestinationDefinition
     try {
       LocalDefinitionsProvider provider = new LocalDefinitionsProvider(LocalDefinitionsProvider.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS);
       return provider.getDestinationDefinitions().stream()
-              .filter(definition -> imageNameWithoutTag.equalsIgnoreCase(definition.getDockerRepository()))
-              .findFirst();
+          .filter(definition -> imageNameWithoutTag.equalsIgnoreCase(definition.getDockerRepository()))
+          .findFirst();
     } catch (IOException e) {
       return Optional.empty();
     }
@@ -140,9 +140,9 @@ private Optional getOptionalDestinationDefinition
 
   protected String getNormalizationImageName() {
     return getOptionalDestinationDefinitionFromProvider(getImageNameWithoutTag())
-      .map(standardDestinationDefinition -> standardDestinationDefinition.getNormalizationRepository() + ":"
-          + standardDestinationDefinition.getNormalizationTag())
-      .orElse(null);
+        .map(standardDestinationDefinition -> standardDestinationDefinition.getNormalizationRepository() + ":"
+            + standardDestinationDefinition.getNormalizationTag())
+        .orElse(null);
   }
 
   /**
@@ -226,13 +226,15 @@ protected boolean implementsAppend() throws WorkerException {
 
   protected boolean normalizationFromDefinition() {
     return getOptionalDestinationDefinitionFromProvider(getImageNameWithoutTag())
-        .map(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getNormalizationRepository()) && Objects.nonNull(standardDestinationDefinition.getNormalizationTag()))
+        .map(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getNormalizationRepository())
+            && Objects.nonNull(standardDestinationDefinition.getNormalizationTag()))
         .orElse(false);
   }
 
   protected boolean dbtFromDefinition() {
     return getOptionalDestinationDefinitionFromProvider(getImageNameWithoutTag())
-        .map(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getSupportsDbt()) && standardDestinationDefinition.getSupportsDbt())
+        .map(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getSupportsDbt())
+            && standardDestinationDefinition.getSupportsDbt())
         .orElse(false);
   }
 

From 48077f77f016a5742872734248126838830e2ba8 Mon Sep 17 00:00:00 2001
From: Andrii Korotkov 
Date: Mon, 5 Dec 2022 11:02:29 +0200
Subject: [PATCH 16/34] fixed code style and removed tests data

---
 .../connectors/destination-mysql/src/main/resources/spec.json    | 1 +
 .../destination/mysql/MySQLDestinationAcceptanceTest.java        | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/airbyte-integrations/connectors/destination-mysql/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-mysql/src/main/resources/spec.json
index 080158fc140e..7b068ddc74e6 100644
--- a/airbyte-integrations/connectors/destination-mysql/src/main/resources/spec.json
+++ b/airbyte-integrations/connectors/destination-mysql/src/main/resources/spec.json
@@ -2,6 +2,7 @@
   "documentationUrl": "https://docs.airbyte.com/integrations/destinations/mysql",
   "supportsIncremental": true,
   "supportsNormalization": true,
+  "supportsDBT": true,
   "supported_destination_sync_modes": ["overwrite", "append"],
   "connectionSpecification": {
     "$schema": "http://json-schema.org/draft-07/schema#",
diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java
index 0d88daceab55..b720ea006c3c 100644
--- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java
+++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java
@@ -50,7 +50,6 @@ protected String getImageName() {
     return "airbyte/destination-mysql:dev";
   }
 
-
   @Override
   protected boolean supportsDBT() {
     return true;

From ccc42c7d4f5bb3e3f8f8c370047ba4190722e65d Mon Sep 17 00:00:00 2001
From: Andrii Korotkov 
Date: Mon, 5 Dec 2022 12:22:26 +0200
Subject: [PATCH 17/34] updated JobErrorReporterTest.java

---
 .../persistence/job/errorreporter/JobErrorReporter.java       | 3 ++-
 .../persistence/job/errorreporter/JobErrorReporterTest.java   | 4 +++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/JobErrorReporter.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/JobErrorReporter.java
index a931e97f1274..b0a0aff5540f 100644
--- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/JobErrorReporter.java
+++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/JobErrorReporter.java
@@ -6,6 +6,7 @@
 
 import com.google.common.collect.ImmutableSet;
 import edu.umd.cs.findbugs.annotations.Nullable;
+import io.airbyte.commons.docker.DockerUtils;
 import io.airbyte.commons.lang.Exceptions;
 import io.airbyte.commons.map.MoreMaps;
 import io.airbyte.config.AttemptFailureSummary;
@@ -116,7 +117,7 @@ public void reportSyncJobFailure(final UUID connectionId, final AttemptFailureSu
               getNormalizationMetadata(destinationDefinition.getNormalizationRepository()),
               prefixConnectorMetadataKeys(getSourceMetadata(sourceDefinition), "source"),
               getDestinationMetadata(destinationDefinition));
-          final String dockerImage = String.format("%s:%s", destinationDefinition.getNormalizationRepository(),
+          final String dockerImage = DockerUtils.getTaggedImageName(destinationDefinition.getNormalizationRepository(),
               destinationDefinition.getNormalizationTag());
 
           reportJobFailureReason(workspace, failureReason, dockerImage, metadata);
diff --git a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/JobErrorReporterTest.java b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/JobErrorReporterTest.java
index f47b2996adad..6f71d8f98aa4 100644
--- a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/JobErrorReporterTest.java
+++ b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/JobErrorReporterTest.java
@@ -38,7 +38,7 @@ class JobErrorReporterTest {
   private static final DeploymentMode DEPLOYMENT_MODE = DeploymentMode.OSS;
   private static final String AIRBYTE_VERSION = "0.1.40";
   private static final String NORMALIZATION_IMAGE = "airbyte/normalization";
-  private static final String NORMALIZATION_VERSION = "0.2.18";
+  private static final String NORMALIZATION_VERSION = "0.2.24";
   private static final UUID SOURCE_DEFINITION_ID = UUID.randomUUID();
   private static final String SOURCE_DEFINITION_NAME = "stripe";
   private static final String SOURCE_DOCKER_REPOSITORY = "airbyte/source-stripe";
@@ -138,6 +138,8 @@ void testReportSyncJobFailure() {
             .withDockerRepository(DESTINATION_DOCKER_REPOSITORY)
             .withReleaseStage(DESTINATION_RELEASE_STAGE)
             .withDestinationDefinitionId(DESTINATION_DEFINITION_ID)
+            .withNormalizationRepository(NORMALIZATION_IMAGE)
+            .withNormalizationTag(NORMALIZATION_VERSION)
             .withName(DESTINATION_DEFINITION_NAME));
 
     final StandardWorkspace mWorkspace = Mockito.mock(StandardWorkspace.class);

From 57f508798d20c7d52a772456ad51f48e77e70924 Mon Sep 17 00:00:00 2001
From: Andrii Korotkov 
Date: Mon, 5 Dec 2022 14:39:51 +0200
Subject: [PATCH 18/34] updated JobErrorReporterTest.java

---
 .../normalization/DefaultNormalizationRunner.java  |  2 +-
 .../DefaultNormalizationRunnerTest.java            | 14 +++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
index dd9dc8b30d1f..3680395a29f7 100644
--- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
+++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
@@ -63,7 +63,7 @@ public DefaultNormalizationRunner(final ProcessFactory processFactory,
 
   private String getDestinationTypeFromNormalizationImage(final String normalizationImage) {
     String imageNameWithoutTag = normalizationImage.contains(":") ? normalizationImage.split(":")[0] : normalizationImage;
-    return "airbyte/normalization".equalsIgnoreCase(imageNameWithoutTag) ? "normalization"
+    return "airbyte/normalization".equalsIgnoreCase(imageNameWithoutTag) ? "bigquery"
         : imageNameWithoutTag.replace("airbyte/normalization-", "");
   }
 
diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
index dbaf63d7ebd4..49a299e705cc 100644
--- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
+++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
@@ -112,7 +112,7 @@ public void tearDown() throws IOException {
 
   @Test
   void test() throws Exception {
-    final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.buildImage(NORMALIZATION_IMAGE, NORMALIZATION_TAG));
+    final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG));
 
     when(process.exitValue()).thenReturn(0);
 
@@ -122,7 +122,7 @@ void test() throws Exception {
   @Test
   void testLog() throws Exception {
 
-    final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.buildImage(NORMALIZATION_IMAGE, NORMALIZATION_TAG));
+    final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG));
 
     when(process.exitValue()).thenReturn(0);
 
@@ -143,7 +143,7 @@ void testLog() throws Exception {
   void testClose() throws Exception {
     when(process.isAlive()).thenReturn(true).thenReturn(false);
 
-    final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.buildImage(NORMALIZATION_IMAGE, NORMALIZATION_TAG));
+    final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG));
     runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements());
     runner.close();
 
@@ -154,7 +154,7 @@ void testClose() throws Exception {
   void testFailure() throws Exception {
     when(process.exitValue()).thenReturn(1);
 
-    final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.buildImage(NORMALIZATION_IMAGE, NORMALIZATION_TAG));
+    final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG));
     assertFalse(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements()));
 
     verify(process).waitFor();
@@ -174,7 +174,7 @@ void testFailureWithTraceMessage() throws Exception {
                                     """.replace("\n", "");
     when(process.getInputStream()).thenReturn(new ByteArrayInputStream(errorTraceString.getBytes(StandardCharsets.UTF_8)));
 
-    final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.buildImage(NORMALIZATION_IMAGE, NORMALIZATION_TAG));
+    final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG));
     assertFalse(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements()));
 
     assertEquals(1, runner.getTraceMessages().count());
@@ -199,7 +199,7 @@ void testFailureWithDbtError() throws Exception {
                                   """;
     when(process.getInputStream()).thenReturn(new ByteArrayInputStream(dbtErrorString.getBytes(StandardCharsets.UTF_8)));
 
-    final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.buildImage(NORMALIZATION_IMAGE, NORMALIZATION_TAG));
+    final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG));
     assertFalse(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements()));
 
     assertEquals(1, runner.getTraceMessages().count());
@@ -219,7 +219,7 @@ void testFailureWithDbtErrorJsonFormat() throws Exception {
         """;
     when(process.getInputStream()).thenReturn(new ByteArrayInputStream(dbtErrorString.getBytes(StandardCharsets.UTF_8)));
 
-    final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.buildImage(NORMALIZATION_IMAGE, NORMALIZATION_TAG));
+    final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG));
     assertFalse(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements()));
 
     assertEquals(1, runner.getTraceMessages().count());

From 413e91bd21ed5a26268c56bf04deb9f656e63561 Mon Sep 17 00:00:00 2001
From: Andrii Korotkov 
Date: Tue, 6 Dec 2022 13:05:45 +0200
Subject: [PATCH 19/34] fixed remarks

---
 .../workers/normalization/DefaultNormalizationRunnerTest.java  | 2 +-
 .../standardtest/destination/DestinationAcceptanceTest.java    | 3 ++-
 .../SnowflakeInternalStagingDestinationAcceptanceTest.java     | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
index 49a299e705cc..f208acb6ca1b 100644
--- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
+++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
@@ -85,7 +85,7 @@ void setup() throws IOException, WorkerException {
         WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME, Jsons.serialize(catalog));
 
     when(processFactory.create(AirbyteIntegrationLauncher.NORMALIZE_STEP, JOB_ID, JOB_ATTEMPT, jobRoot,
-        "airbyte/normalization", false, files, null,
+        NORMALIZATION_IMAGE, false, files, null,
         workerConfigs.getResourceRequirements(),
         Map.of(AirbyteIntegrationLauncher.JOB_TYPE, AirbyteIntegrationLauncher.SYNC_JOB, AirbyteIntegrationLauncher.SYNC_STEP,
             AirbyteIntegrationLauncher.NORMALIZE_STEP),
diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
index 2d701a37d1e0..4ec474f79ca8 100644
--- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
+++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
@@ -99,6 +99,7 @@
 public abstract class DestinationAcceptanceTest {
 
   private static final Random RANDOM = new Random();
+  private static final String NORMALIZATION_VERSION = "dev";
 
   private static final String JOB_ID = "0";
   private static final int JOB_ATTEMPT = 0;
@@ -141,7 +142,7 @@ private Optional getOptionalDestinationDefinition
   protected String getNormalizationImageName() {
     return getOptionalDestinationDefinitionFromProvider(getImageNameWithoutTag())
         .map(standardDestinationDefinition -> standardDestinationDefinition.getNormalizationRepository() + ":"
-            + standardDestinationDefinition.getNormalizationTag())
+            + NORMALIZATION_VERSION)
         .orElse(null);
   }
 
diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java
index e73187ed00a8..dd89ebc2d955 100644
--- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java
+++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java
@@ -43,7 +43,7 @@ public void testSyncWithNormalizationWithKeyPairEncrypt(final String messagesFil
   }
 
   private void testSyncWithNormalizationWithKeyPairAuth(String messagesFilename, String catalogFilename, String configName) throws Exception {
-    if (!normalizationFromSpec()) {
+    if (!normalizationFromDefinition()) {
       return;
     }
 

From d2ac3aee04f7e26e4d2824181c429a6644d57d82 Mon Sep 17 00:00:00 2001
From: AndriiKorotkov 
Date: Thu, 8 Dec 2022 13:56:15 +0200
Subject: [PATCH 20/34] added integration type field to the
 dectination_definition file and actor_definition table

---
 .../DefaultNormalizationRunner.java           | 11 ++----
 .../DefaultNormalizationRunnerTest.java       | 22 ++++++++----
 .../types/StandardDestinationDefinition.yaml  |  2 ++
 .../config/persistence/ConfigWriter.java      |  2 ++
 .../config/persistence/DbConverter.java       |  2 ++
 .../seed/destination_definitions.yaml         | 10 ++++++
 .../orchestrator/DbtJobOrchestrator.java      |  3 +-
 .../NormalizationJobOrchestrator.java         |  3 +-
 ...__AddIntegrationTypeToActorDefinition.java | 36 +++++++++++++++++++
 .../normalized_tables_schema.txt              |  1 +
 .../configs_database/schema_dump.txt          |  1 +
 .../DestinationAcceptanceTest.java            | 18 +++++++---
 .../DestinationDefinitionsHandler.java        |  1 +
 .../IntegrationLauncherConfig.yaml            |  2 ++
 .../activities/GenerateInputActivityImpl.java |  6 ++--
 .../sync/DbtTransformationActivityImpl.java   |  3 +-
 .../sync/NormalizationActivityImpl.java       |  3 +-
 17 files changed, 101 insertions(+), 25 deletions(-)
 create mode 100644 airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_22_001__AddIntegrationTypeToActorDefinition.java

diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
index 3680395a29f7..b2e8923966b8 100644
--- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
+++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
@@ -55,16 +55,11 @@ public class DefaultNormalizationRunner implements NormalizationRunner {
   private Process process = null;
 
   public DefaultNormalizationRunner(final ProcessFactory processFactory,
-                                    final String normalizationImage) {
+                                    final String normalizationImage,
+                                    final String destinationType) {
     this.processFactory = processFactory;
     this.normalizationImageName = normalizationImage;
-    this.destinationType = getDestinationTypeFromNormalizationImage(normalizationImage);
-  }
-
-  private String getDestinationTypeFromNormalizationImage(final String normalizationImage) {
-    String imageNameWithoutTag = normalizationImage.contains(":") ? normalizationImage.split(":")[0] : normalizationImage;
-    return "airbyte/normalization".equalsIgnoreCase(imageNameWithoutTag) ? "bigquery"
-        : imageNameWithoutTag.replace("airbyte/normalization-", "");
+    this.destinationType = destinationType;
   }
 
   @Override
diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
index f208acb6ca1b..2443b3693515 100644
--- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
+++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
@@ -51,6 +51,7 @@ class DefaultNormalizationRunnerTest {
 
   private static final String NORMALIZATION_IMAGE = "airbyte/normalization";
   private static final String NORMALIZATION_TAG = "0.2.24";
+  private static final String INTEGRATION_TYPE = "postgres";
 
   private static Path logJobRoot;
 
@@ -112,7 +113,8 @@ public void tearDown() throws IOException {
 
   @Test
   void test() throws Exception {
-    final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG));
+    final NormalizationRunner runner =
+        new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), INTEGRATION_TYPE);
 
     when(process.exitValue()).thenReturn(0);
 
@@ -122,7 +124,8 @@ void test() throws Exception {
   @Test
   void testLog() throws Exception {
 
-    final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG));
+    final NormalizationRunner runner =
+        new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), INTEGRATION_TYPE);
 
     when(process.exitValue()).thenReturn(0);
 
@@ -143,7 +146,8 @@ void testLog() throws Exception {
   void testClose() throws Exception {
     when(process.isAlive()).thenReturn(true).thenReturn(false);
 
-    final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG));
+    final NormalizationRunner runner =
+        new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), INTEGRATION_TYPE);
     runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements());
     runner.close();
 
@@ -154,7 +158,8 @@ void testClose() throws Exception {
   void testFailure() throws Exception {
     when(process.exitValue()).thenReturn(1);
 
-    final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG));
+    final NormalizationRunner runner =
+        new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), INTEGRATION_TYPE);
     assertFalse(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements()));
 
     verify(process).waitFor();
@@ -174,7 +179,8 @@ void testFailureWithTraceMessage() throws Exception {
                                     """.replace("\n", "");
     when(process.getInputStream()).thenReturn(new ByteArrayInputStream(errorTraceString.getBytes(StandardCharsets.UTF_8)));
 
-    final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG));
+    final NormalizationRunner runner =
+        new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), INTEGRATION_TYPE);
     assertFalse(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements()));
 
     assertEquals(1, runner.getTraceMessages().count());
@@ -199,7 +205,8 @@ void testFailureWithDbtError() throws Exception {
                                   """;
     when(process.getInputStream()).thenReturn(new ByteArrayInputStream(dbtErrorString.getBytes(StandardCharsets.UTF_8)));
 
-    final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG));
+    final NormalizationRunner runner =
+        new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), INTEGRATION_TYPE);
     assertFalse(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements()));
 
     assertEquals(1, runner.getTraceMessages().count());
@@ -219,7 +226,8 @@ void testFailureWithDbtErrorJsonFormat() throws Exception {
         """;
     when(process.getInputStream()).thenReturn(new ByteArrayInputStream(dbtErrorString.getBytes(StandardCharsets.UTF_8)));
 
-    final NormalizationRunner runner = new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG));
+    final NormalizationRunner runner =
+        new DefaultNormalizationRunner(processFactory, DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), INTEGRATION_TYPE);
     assertFalse(runner.normalize(JOB_ID, JOB_ATTEMPT, jobRoot, config, catalog, workerConfigs.getResourceRequirements()));
 
     assertEquals(1, runner.getTraceMessages().count());
diff --git a/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml b/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml
index 4c51a6b3239c..bc4a835cc8b5 100644
--- a/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml
+++ b/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml
@@ -64,3 +64,5 @@ properties:
     type: string
   supportsDbt:
     type: boolean
+  integrationType:
+    type: string
diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java
index cb836f63fbb9..52542308b784 100644
--- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java
+++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java
@@ -178,6 +178,7 @@ static void writeStandardDestinationDefinition(final List runJob() throws Exception {
         new DbtTransformationRunner(
             processFactory, new DefaultNormalizationRunner(
                 processFactory,
-                destinationLauncherConfig.getNormalizationDockerImage())));
+                destinationLauncherConfig.getNormalizationDockerImage(),
+                destinationLauncherConfig.getIntegrationType())));
 
     log.info("Running dbt worker...");
     final Path jobRoot = TemporalUtils.getJobRoot(configs.getWorkspaceRoot(),
diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/NormalizationJobOrchestrator.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/NormalizationJobOrchestrator.java
index 5035b11a5587..cb6c2f26102f 100644
--- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/NormalizationJobOrchestrator.java
+++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/NormalizationJobOrchestrator.java
@@ -72,7 +72,8 @@ public Optional runJob() throws Exception {
         Math.toIntExact(jobRunConfig.getAttemptId()),
         new DefaultNormalizationRunner(
             processFactory,
-            destinationLauncherConfig.getNormalizationDockerImage()),
+            destinationLauncherConfig.getNormalizationDockerImage(),
+            destinationLauncherConfig.getIntegrationType()),
         configs.getWorkerEnvironment());
 
     log.info("Running normalization worker...");
diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_22_001__AddIntegrationTypeToActorDefinition.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_22_001__AddIntegrationTypeToActorDefinition.java
new file mode 100644
index 000000000000..007a539dfc2a
--- /dev/null
+++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_22_001__AddIntegrationTypeToActorDefinition.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.db.instance.configs.migrations;
+
+import org.flywaydb.core.api.migration.BaseJavaMigration;
+import org.flywaydb.core.api.migration.Context;
+import org.jooq.DSLContext;
+import org.jooq.impl.DSL;
+import org.jooq.impl.SQLDataType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+// TODO: update migration description in the class name
+public class V0_40_22_001__AddIntegrationTypeToActorDefinition extends BaseJavaMigration {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(
+      V0_40_22_001__AddIntegrationTypeToActorDefinition.class);
+
+  @Override
+  public void migrate(final Context context) throws Exception {
+    LOGGER.info("Running migration: {}", this.getClass().getSimpleName());
+    final DSLContext ctx = DSL.using(context.getConnection());
+    addIntegrationTypeColumn(ctx);
+  }
+
+  static void addIntegrationTypeColumn(final DSLContext ctx) {
+    ctx.alterTable("actor_definition")
+        .addColumnIfNotExists(DSL.field(
+            "integration_type",
+            SQLDataType.VARCHAR(255).nullable(true)))
+        .execute();
+  }
+
+}
diff --git a/airbyte-db/db-lib/src/main/resources/configs_database/normalized_tables_schema.txt b/airbyte-db/db-lib/src/main/resources/configs_database/normalized_tables_schema.txt
index fa9c708d43d9..c4b9a68f331a 100644
--- a/airbyte-db/db-lib/src/main/resources/configs_database/normalized_tables_schema.txt
+++ b/airbyte-db/db-lib/src/main/resources/configs_database/normalized_tables_schema.txt
@@ -91,6 +91,7 @@ Referenced by:
  normalization_repository   | character varying(255)   |           |          |
  normalization_tag          | character varying(255)   |           |          |
  supports_dbt               | boolean                  |           |          |
+ integration_type           | character varying(255)   |           |          |
 Indexes:
     "actor_definition_pkey" PRIMARY KEY, btree (id)
 Referenced by:
diff --git a/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt b/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt
index 33298d85df3b..be48e612c0e5 100644
--- a/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt
+++ b/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt
@@ -57,6 +57,7 @@ create table "public"."actor_definition"(
   "normalization_repository" varchar(255) null,
   "normalization_tag" varchar(255) null,
   "supports_dbt" bool null,
+  "integration_type" varchar(255) null,
   constraint "actor_definition_pkey"
     primary key ("id")
 );
diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
index 4ec474f79ca8..3e501d9834ca 100644
--- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
+++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
@@ -239,6 +239,12 @@ protected boolean dbtFromDefinition() {
         .orElse(false);
   }
 
+  protected String getIntegrationType() {
+    return getOptionalDestinationDefinitionFromProvider(getImageNameWithoutTag())
+        .map(StandardDestinationDefinition::getIntegrationType)
+        .orElse(null);
+  }
+
   /**
    * Detects if a destination implements append dedup mode from the spec.json that should include
    * 'supportedDestinationSyncMode'
@@ -563,7 +569,8 @@ public void specNormalizationValueShouldBeCorrect() throws Exception {
       try {
         new DefaultNormalizationRunner(
             processFactory,
-            getNormalizationImageName());
+            getNormalizationImageName(),
+            getIntegrationType());
         normalizationRunnerFactorySupportsDestinationImage = true;
       } catch (final IllegalStateException e) {
         normalizationRunnerFactorySupportsDestinationImage = false;
@@ -868,7 +875,8 @@ public void testCustomDbtTransformations() throws Exception {
     final DbtTransformationRunner runner = new DbtTransformationRunner(processFactory,
         new DefaultNormalizationRunner(
             processFactory,
-            getNormalizationImageName()));
+            getNormalizationImageName(),
+            getIntegrationType()));
     runner.start();
     final Path transformationRoot = Files.createDirectories(jobRoot.resolve("transform"));
     final OperatorDbt dbtConfig = new OperatorDbt()
@@ -943,7 +951,8 @@ void testCustomDbtTransformationsFailure() throws Exception {
     final DbtTransformationRunner runner = new DbtTransformationRunner(processFactory,
         new DefaultNormalizationRunner(
             processFactory,
-            getNormalizationImageName()));
+            getNormalizationImageName(),
+            getIntegrationType()));
     runner.start();
     final Path transformationRoot = Files.createDirectories(jobRoot.resolve("transform"));
     final OperatorDbt dbtConfig = new OperatorDbt()
@@ -1289,7 +1298,8 @@ private List runSync(
 
     final NormalizationRunner runner = new DefaultNormalizationRunner(
         processFactory,
-        getNormalizationImageName());
+        getNormalizationImageName(),
+        getIntegrationType());
     runner.start();
     final Path normalizationRoot = Files.createDirectories(jobRoot.resolve("normalize"));
     if (!runner.normalize(JOB_ID, JOB_ATTEMPT, normalizationRoot,
diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java
index 0a995f4ba717..0708a2c9117c 100644
--- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java
+++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java
@@ -267,6 +267,7 @@ public DestinationDefinitionRead updateDestinationDefinition(final DestinationDe
         .withNormalizationRepository(currentDestination.getNormalizationRepository())
         .withNormalizationTag(currentDestination.getNormalizationTag())
         .withSupportsDbt(currentDestination.getSupportsDbt())
+        .withIntegrationType(currentDestination.getIntegrationType())
         .withSpec(spec)
         .withProtocolVersion(airbyteProtocolVersion.serialize())
         .withTombstone(currentDestination.getTombstone())
diff --git a/airbyte-worker-models/src/main/resources/workers_models/IntegrationLauncherConfig.yaml b/airbyte-worker-models/src/main/resources/workers_models/IntegrationLauncherConfig.yaml
index d3c24d7aa12f..c3dc446afa4c 100644
--- a/airbyte-worker-models/src/main/resources/workers_models/IntegrationLauncherConfig.yaml
+++ b/airbyte-worker-models/src/main/resources/workers_models/IntegrationLauncherConfig.yaml
@@ -21,6 +21,8 @@ properties:
   supportsDbt:
     type: boolean
     default: false
+  integrationType:
+    type: string
   protocolVersion:
     type: object
     existingJavaType: io.airbyte.commons.version.Version
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java
index 8081cedd4dfc..e5a72e559c5c 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java
@@ -95,7 +95,8 @@ public GeneratedJobInput getSyncWorkflowInput(final SyncInput input) {
           .findFirst();
       final String destinationNormalizationDockerImage = optionalDestinationDefinition.map(standardDestinationDefinition -> String.format("%s:%s",
           standardDestinationDefinition.getNormalizationRepository(), standardDestinationDefinition.getNormalizationTag())).orElse(null);
-      final boolean supportDbt = optionalDestinationDefinition.isPresent() ? optionalDestinationDefinition.get().getSupportsDbt() : false;
+      final boolean supportstDbt = optionalDestinationDefinition.isPresent() ? optionalDestinationDefinition.get().getSupportsDbt() : false;
+      final String integrationType = optionalDestinationDefinition.map(StandardDestinationDefinition::getIntegrationType).orElse(null);
 
       final IntegrationLauncherConfig sourceLauncherConfig = new IntegrationLauncherConfig()
           .withJobId(String.valueOf(jobId))
@@ -109,7 +110,8 @@ public GeneratedJobInput getSyncWorkflowInput(final SyncInput input) {
           .withDockerImage(config.getDestinationDockerImage())
           .withProtocolVersion(config.getDestinationProtocolVersion())
           .withNormalizationDockerImage(destinationNormalizationDockerImage)
-          .withSupportsDbt(supportDbt);
+          .withSupportsDbt(supportstDbt)
+          .withIntegrationType(integrationType);
 
       final StandardSyncInput syncInput = new StandardSyncInput()
           .withNamespaceDefinition(config.getNamespaceDefinition())
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java
index 9ad830e05378..cc716a3e8802 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java
@@ -144,7 +144,8 @@ private CheckedSupplier, Exception> getLegacyWork
         new DbtTransformationRunner(
             processFactory, new DefaultNormalizationRunner(
                 processFactory,
-                destinationLauncherConfig.getNormalizationDockerImage())));
+                destinationLauncherConfig.getNormalizationDockerImage(),
+                destinationLauncherConfig.getIntegrationType())));
   }
 
   private CheckedSupplier, Exception> getContainerLauncherWorkerFactory(
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java
index 88e9eb315d84..260644334be4 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java
@@ -152,7 +152,8 @@ private CheckedSupplier, Except
         Math.toIntExact(jobRunConfig.getAttemptId()),
         new DefaultNormalizationRunner(
             processFactory,
-            destinationLauncherConfig.getNormalizationDockerImage()),
+            destinationLauncherConfig.getNormalizationDockerImage(),
+            destinationLauncherConfig.getIntegrationType()),
         workerEnvironment);
   }
 

From afc8afa359bf40399692561aa91c808cacfd5b95 Mon Sep 17 00:00:00 2001
From: AndriiKorotkov 
Date: Thu, 8 Dec 2022 18:39:21 +0200
Subject: [PATCH 21/34] fixed tests

---
 .../java/io/airbyte/bootloader/BootloaderAppTest.java  |  2 +-
 .../normalization/DefaultNormalizationRunner.java      | 10 +++++-----
 .../normalization/DefaultNormalizationRunnerTest.java  |  4 ++--
 .../main/resources/seed/destination_definitions.yaml   |  2 +-
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java
index fbc113f69063..2268e27b3889 100644
--- a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java
+++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java
@@ -142,7 +142,7 @@ void testBootloaderAppBlankDb() throws Exception {
       val configsMigrator = new ConfigsDatabaseMigrator(configDatabase, configsFlyway);
       // this line should change with every new migration
       // to show that you meant to make a new migration to the prod database
-      assertEquals("0.40.18.004", configsMigrator.getLatestMigration().getVersion().getVersion());
+      assertEquals("0.40.22.001", configsMigrator.getLatestMigration().getVersion().getVersion());
 
       val jobsPersistence = new DefaultJobPersistence(jobDatabase);
       assertEquals(VERSION_0330_ALPHA, jobsPersistence.getVersion().get());
diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
index b2e8923966b8..d082b68a9104 100644
--- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
+++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
@@ -45,7 +45,7 @@ public class DefaultNormalizationRunner implements NormalizationRunner {
       .setLogPrefix("normalization")
       .setPrefixColor(Color.GREEN_BACKGROUND);
 
-  private final String destinationType;
+  private final String integrationType;
   private final ProcessFactory processFactory;
   private final String normalizationImageName;
   private final NormalizationAirbyteStreamFactory streamFactory = new NormalizationAirbyteStreamFactory(CONTAINER_LOG_MDC_BUILDER);
@@ -59,7 +59,7 @@ public DefaultNormalizationRunner(final ProcessFactory processFactory,
                                     final String destinationType) {
     this.processFactory = processFactory;
     this.normalizationImageName = normalizationImage;
-    this.destinationType = destinationType;
+    this.integrationType = destinationType;
   }
 
   @Override
@@ -79,12 +79,12 @@ public boolean configureDbt(final String jobId,
     final String gitRepoBranch = dbtConfig.getGitRepoBranch();
     if (Strings.isNullOrEmpty(gitRepoBranch)) {
       return runProcess(jobId, attempt, jobRoot, files, resourceRequirements, "configure-dbt",
-          "--integration-type", destinationType.toLowerCase(),
+          "--integration-type", integrationType.toLowerCase(),
           "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
           "--git-repo", gitRepoUrl);
     } else {
       return runProcess(jobId, attempt, jobRoot, files, resourceRequirements, "configure-dbt",
-          "--integration-type", destinationType.toLowerCase(),
+          "--integration-type", integrationType.toLowerCase(),
           "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
           "--git-repo", gitRepoUrl,
           "--git-branch", gitRepoBranch);
@@ -104,7 +104,7 @@ public boolean normalize(final String jobId,
         WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME, Jsons.serialize(catalog));
 
     return runProcess(jobId, attempt, jobRoot, files, resourceRequirements, "run",
-        "--integration-type", destinationType.toLowerCase(),
+        "--integration-type", integrationType.toLowerCase(),
         "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
         "--catalog", WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME);
   }
diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
index 2443b3693515..2f15d8d868cd 100644
--- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
+++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
@@ -86,14 +86,14 @@ void setup() throws IOException, WorkerException {
         WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME, Jsons.serialize(catalog));
 
     when(processFactory.create(AirbyteIntegrationLauncher.NORMALIZE_STEP, JOB_ID, JOB_ATTEMPT, jobRoot,
-        NORMALIZATION_IMAGE, false, files, null,
+        DockerUtils.getTaggedImageName(NORMALIZATION_IMAGE, NORMALIZATION_TAG), false, files, null,
         workerConfigs.getResourceRequirements(),
         Map.of(AirbyteIntegrationLauncher.JOB_TYPE, AirbyteIntegrationLauncher.SYNC_JOB, AirbyteIntegrationLauncher.SYNC_STEP,
             AirbyteIntegrationLauncher.NORMALIZE_STEP),
         Map.of(),
         Map.of(),
         "run",
-        "--integration-type", "bigquery",
+        "--integration-type", INTEGRATION_TYPE,
         "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
         "--catalog", WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME))
             .thenReturn(process);
diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
index eae6c5f34e4b..a804f9ff0651 100644
--- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
+++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
@@ -250,7 +250,7 @@
   documentationUrl: https://docs.airbyte.com/integrations/destinations/postgres
   icon: postgresql.svg
   releaseStage: alpha
-  normalizationRepository: airbyte/normalization-postgres
+  normalizationRepository: airbyte/normalization
   normalizationTag: 0.2.24
   supportsDbt: true
   integrationType: postgres

From 213290a2baa802f13d3a61f8bbd89f49c0167c83 Mon Sep 17 00:00:00 2001
From: AndriiKorotkov 
Date: Thu, 8 Dec 2022 22:47:29 +0200
Subject: [PATCH 22/34] fixed tests

---
 ...=> V0_40_23_002__AddIntegrationTypeToActorDefinition.java} | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
 rename airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/{V0_40_22_001__AddIntegrationTypeToActorDefinition.java => V0_40_23_002__AddIntegrationTypeToActorDefinition.java} (88%)

diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_22_001__AddIntegrationTypeToActorDefinition.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_23_002__AddIntegrationTypeToActorDefinition.java
similarity index 88%
rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_22_001__AddIntegrationTypeToActorDefinition.java
rename to airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_23_002__AddIntegrationTypeToActorDefinition.java
index 007a539dfc2a..4d168177ead2 100644
--- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_22_001__AddIntegrationTypeToActorDefinition.java
+++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_23_002__AddIntegrationTypeToActorDefinition.java
@@ -13,10 +13,10 @@
 import org.slf4j.LoggerFactory;
 
 // TODO: update migration description in the class name
-public class V0_40_22_001__AddIntegrationTypeToActorDefinition extends BaseJavaMigration {
+public class V0_40_23_002__AddIntegrationTypeToActorDefinition extends BaseJavaMigration {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(
-      V0_40_22_001__AddIntegrationTypeToActorDefinition.class);
+      V0_40_23_002__AddIntegrationTypeToActorDefinition.class);
 
   @Override
   public void migrate(final Context context) throws Exception {

From 040f2a260380c4db0acd81d06a10762f1f2a8d35 Mon Sep 17 00:00:00 2001
From: AndriiKorotkov 
Date: Fri, 9 Dec 2022 10:35:47 +0200
Subject: [PATCH 23/34] fixed minor changes after pulling master changes

---
 .../airbyte/workers/normalization/NormalizationRunnerFactory.java | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java

diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java
deleted file mode 100644
index e69de29bb2d1..000000000000

From 5ea2e14fa0680c87ccc0ecd11674b8e362d6ce9d Mon Sep 17 00:00:00 2001
From: AndriiKorotkov 
Date: Fri, 9 Dec 2022 11:08:04 +0200
Subject: [PATCH 24/34] fixed minor changes after pulling master changes

---
 .../workers/normalization/DefaultNormalizationRunner.java     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
index 1388eddf0fe2..7365efe7bab2 100644
--- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
+++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
@@ -56,10 +56,10 @@ public class DefaultNormalizationRunner implements NormalizationRunner {
 
   public DefaultNormalizationRunner(final ProcessFactory processFactory,
                                     final String normalizationImage,
-                                    final String destinationType) {
+                                    final String integrationType) {
     this.processFactory = processFactory;
     this.normalizationImageName = normalizationImage;
-    this.integrationType = destinationType;
+    this.integrationType = integrationType;
   }
 
   @Override

From 534ea588507b2b1f08fd2237f5a8cf4a06b52c42 Mon Sep 17 00:00:00 2001
From: AndriiKorotkov 
Date: Mon, 12 Dec 2022 13:03:28 +0200
Subject: [PATCH 25/34] renamed integrationType to
 normalizationIntegrationType/ fixed minor remarks

---
 .../DefaultNormalizationRunner.java           | 12 +++++------
 .../types/StandardDestinationDefinition.yaml  |  6 +++++-
 .../config/persistence/ConfigWriter.java      |  4 ++--
 .../config/persistence/DbConverter.java       |  4 ++--
 .../seed/destination_definitions.yaml         | 20 +++++++++----------
 .../orchestrator/DbtJobOrchestrator.java      |  2 +-
 .../NormalizationJobOrchestrator.java         |  2 +-
 ...tionIntegrationTypeToActorDefinition.java} |  6 +++---
 .../DestinationAcceptanceTest.java            | 12 +++++------
 .../DestinationDefinitionsHandler.java        |  2 +-
 .../IntegrationLauncherConfig.yaml            |  2 +-
 .../activities/GenerateInputActivityImpl.java |  5 +++--
 .../sync/DbtTransformationActivityImpl.java   |  2 +-
 .../sync/NormalizationActivityImpl.java       |  2 +-
 14 files changed, 43 insertions(+), 38 deletions(-)
 rename airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/{V0_40_23_002__AddIntegrationTypeToActorDefinition.java => V0_40_23_002__AddNormalizationIntegrationTypeToActorDefinition.java} (81%)

diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
index 7365efe7bab2..e96f9c81d403 100644
--- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
+++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java
@@ -45,7 +45,7 @@ public class DefaultNormalizationRunner implements NormalizationRunner {
       .setLogPrefix("normalization")
       .setPrefixColor(Color.GREEN_BACKGROUND);
 
-  private final String integrationType;
+  private final String normalizationIntegrationType;
   private final ProcessFactory processFactory;
   private final String normalizationImageName;
   private final NormalizationAirbyteStreamFactory streamFactory = new NormalizationAirbyteStreamFactory(CONTAINER_LOG_MDC_BUILDER);
@@ -56,10 +56,10 @@ public class DefaultNormalizationRunner implements NormalizationRunner {
 
   public DefaultNormalizationRunner(final ProcessFactory processFactory,
                                     final String normalizationImage,
-                                    final String integrationType) {
+                                    final String normalizationIntegrationType) {
     this.processFactory = processFactory;
     this.normalizationImageName = normalizationImage;
-    this.integrationType = integrationType;
+    this.normalizationIntegrationType = normalizationIntegrationType;
   }
 
   @Override
@@ -79,12 +79,12 @@ public boolean configureDbt(final String jobId,
     final String gitRepoBranch = dbtConfig.getGitRepoBranch();
     if (Strings.isNullOrEmpty(gitRepoBranch)) {
       return runProcess(jobId, attempt, jobRoot, files, resourceRequirements, "configure-dbt",
-          "--integration-type", integrationType.toLowerCase(),
+          "--integration-type", normalizationIntegrationType.toLowerCase(),
           "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
           "--git-repo", gitRepoUrl);
     } else {
       return runProcess(jobId, attempt, jobRoot, files, resourceRequirements, "configure-dbt",
-          "--integration-type", integrationType.toLowerCase(),
+          "--integration-type", normalizationIntegrationType.toLowerCase(),
           "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
           "--git-repo", gitRepoUrl,
           "--git-branch", gitRepoBranch);
@@ -104,7 +104,7 @@ public boolean normalize(final String jobId,
         WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME, Jsons.serialize(catalog));
 
     return runProcess(jobId, attempt, jobRoot, files, resourceRequirements, "run",
-        "--integration-type", integrationType.toLowerCase(),
+        "--integration-type", normalizationIntegrationType.toLowerCase(),
         "--config", WorkerConstants.DESTINATION_CONFIG_JSON_FILENAME,
         "--catalog", WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME);
   }
diff --git a/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml b/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml
index bc4a835cc8b5..309267aa93d7 100644
--- a/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml
+++ b/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml
@@ -60,9 +60,13 @@ properties:
     description: the Airbyte Protocol version supported by the connector
   normalizationRepository:
     type: string
+    description: an optional field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.
   normalizationTag:
     type: string
+    description: an optional field indicating the tag of the repository to be used for normalization.
   supportsDbt:
     type: boolean
-  integrationType:
+    description: an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used.
+  normalizationIntegrationType:
     type: string
+    description: an optional field indicating the type of integration dialect to use for normalization. If the value is NULL, the default dialect will be used.
diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java
index 52542308b784..45c61f0b2f33 100644
--- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java
+++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java
@@ -178,7 +178,7 @@ static void writeStandardDestinationDefinition(final List runJob() throws Exception {
             processFactory, new DefaultNormalizationRunner(
                 processFactory,
                 destinationLauncherConfig.getNormalizationDockerImage(),
-                destinationLauncherConfig.getIntegrationType())));
+                destinationLauncherConfig.getNormalizationIntegrationType())));
 
     log.info("Running dbt worker...");
     final Path jobRoot = TemporalUtils.getJobRoot(configs.getWorkspaceRoot(),
diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/NormalizationJobOrchestrator.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/NormalizationJobOrchestrator.java
index cb6c2f26102f..5c0b2e178ad2 100644
--- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/NormalizationJobOrchestrator.java
+++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/NormalizationJobOrchestrator.java
@@ -73,7 +73,7 @@ public Optional runJob() throws Exception {
         new DefaultNormalizationRunner(
             processFactory,
             destinationLauncherConfig.getNormalizationDockerImage(),
-            destinationLauncherConfig.getIntegrationType()),
+            destinationLauncherConfig.getNormalizationIntegrationType()),
         configs.getWorkerEnvironment());
 
     log.info("Running normalization worker...");
diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_23_002__AddIntegrationTypeToActorDefinition.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_23_002__AddNormalizationIntegrationTypeToActorDefinition.java
similarity index 81%
rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_23_002__AddIntegrationTypeToActorDefinition.java
rename to airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_23_002__AddNormalizationIntegrationTypeToActorDefinition.java
index 4d168177ead2..c1cb62d2f4c8 100644
--- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_23_002__AddIntegrationTypeToActorDefinition.java
+++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_23_002__AddNormalizationIntegrationTypeToActorDefinition.java
@@ -13,10 +13,10 @@
 import org.slf4j.LoggerFactory;
 
 // TODO: update migration description in the class name
-public class V0_40_23_002__AddIntegrationTypeToActorDefinition extends BaseJavaMigration {
+public class V0_40_23_002__AddNormalizationIntegrationTypeToActorDefinition extends BaseJavaMigration {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(
-      V0_40_23_002__AddIntegrationTypeToActorDefinition.class);
+      V0_40_23_002__AddNormalizationIntegrationTypeToActorDefinition.class);
 
   @Override
   public void migrate(final Context context) throws Exception {
@@ -28,7 +28,7 @@ public void migrate(final Context context) throws Exception {
   static void addIntegrationTypeColumn(final DSLContext ctx) {
     ctx.alterTable("actor_definition")
         .addColumnIfNotExists(DSL.field(
-            "integration_type",
+            "normalization_integration_type",
             SQLDataType.VARCHAR(255).nullable(true)))
         .execute();
   }
diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
index 99429b629d4b..6e07d54d60aa 100644
--- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
+++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
@@ -239,9 +239,9 @@ protected boolean dbtFromDefinition() {
         .orElse(false);
   }
 
-  protected String getIntegrationType() {
+  protected String getNormalizationIntegrationType() {
     return getOptionalDestinationDefinitionFromProvider(getImageNameWithoutTag())
-        .map(StandardDestinationDefinition::getIntegrationType)
+        .map(StandardDestinationDefinition::getNormalizationIntegrationType)
         .orElse(null);
   }
 
@@ -570,7 +570,7 @@ public void specNormalizationValueShouldBeCorrect() throws Exception {
         new DefaultNormalizationRunner(
             processFactory,
             getNormalizationImageName(),
-            getIntegrationType());
+            getNormalizationIntegrationType());
         normalizationRunnerFactorySupportsDestinationImage = true;
       } catch (final IllegalStateException e) {
         normalizationRunnerFactorySupportsDestinationImage = false;
@@ -876,7 +876,7 @@ public void testCustomDbtTransformations() throws Exception {
         new DefaultNormalizationRunner(
             processFactory,
             getNormalizationImageName(),
-            getIntegrationType()));
+            getNormalizationIntegrationType()));
     runner.start();
     final Path transformationRoot = Files.createDirectories(jobRoot.resolve("transform"));
     final OperatorDbt dbtConfig = new OperatorDbt()
@@ -952,7 +952,7 @@ void testCustomDbtTransformationsFailure() throws Exception {
         new DefaultNormalizationRunner(
             processFactory,
             getNormalizationImageName(),
-            getIntegrationType()));
+            getNormalizationIntegrationType()));
     runner.start();
     final Path transformationRoot = Files.createDirectories(jobRoot.resolve("transform"));
     final OperatorDbt dbtConfig = new OperatorDbt()
@@ -1299,7 +1299,7 @@ private List runSync(
     final NormalizationRunner runner = new DefaultNormalizationRunner(
         processFactory,
         getNormalizationImageName(),
-        getIntegrationType());
+        getNormalizationIntegrationType());
     runner.start();
     final Path normalizationRoot = Files.createDirectories(jobRoot.resolve("normalize"));
     if (!runner.normalize(JOB_ID, JOB_ATTEMPT, normalizationRoot,
diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java
index d70742f887f8..98ab0e07b5d8 100644
--- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java
+++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java
@@ -254,7 +254,7 @@ public DestinationDefinitionRead updateDestinationDefinition(final DestinationDe
         .withNormalizationRepository(currentDestination.getNormalizationRepository())
         .withNormalizationTag(currentDestination.getNormalizationTag())
         .withSupportsDbt(currentDestination.getSupportsDbt())
-        .withIntegrationType(currentDestination.getIntegrationType())
+        .withNormalizationIntegrationType(currentDestination.getNormalizationIntegrationType())
         .withSpec(spec)
         .withProtocolVersion(airbyteProtocolVersion.serialize())
         .withTombstone(currentDestination.getTombstone())
diff --git a/airbyte-worker-models/src/main/resources/workers_models/IntegrationLauncherConfig.yaml b/airbyte-worker-models/src/main/resources/workers_models/IntegrationLauncherConfig.yaml
index 9bd4abf2ef6a..e8035d05b6ab 100644
--- a/airbyte-worker-models/src/main/resources/workers_models/IntegrationLauncherConfig.yaml
+++ b/airbyte-worker-models/src/main/resources/workers_models/IntegrationLauncherConfig.yaml
@@ -21,7 +21,7 @@ properties:
   supportsDbt:
     type: boolean
     default: false
-  integrationType:
+  normalizationIntegrationType:
     type: string
   protocolVersion:
     type: object
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java
index 02c46c8f3a03..0b5a72eb25be 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java
@@ -98,7 +98,8 @@ public GeneratedJobInput getSyncWorkflowInput(final SyncInput input) {
       final String destinationNormalizationDockerImage = optionalDestinationDefinition.map(standardDestinationDefinition -> String.format("%s:%s",
           standardDestinationDefinition.getNormalizationRepository(), standardDestinationDefinition.getNormalizationTag())).orElse(null);
       final boolean supportstDbt = optionalDestinationDefinition.isPresent() ? optionalDestinationDefinition.get().getSupportsDbt() : false;
-      final String integrationType = optionalDestinationDefinition.map(StandardDestinationDefinition::getIntegrationType).orElse(null);
+      final String normalizationIntegrationType =
+          optionalDestinationDefinition.map(StandardDestinationDefinition::getNormalizationIntegrationType).orElse(null);
 
       final IntegrationLauncherConfig sourceLauncherConfig = new IntegrationLauncherConfig()
           .withJobId(String.valueOf(jobId))
@@ -115,7 +116,7 @@ public GeneratedJobInput getSyncWorkflowInput(final SyncInput input) {
           .withIsCustomConnector(config.getIsDestinationCustomConnector())
           .withNormalizationDockerImage(destinationNormalizationDockerImage)
           .withSupportsDbt(supportstDbt)
-          .withIntegrationType(integrationType);
+          .withNormalizationIntegrationType(normalizationIntegrationType);
 
       final StandardSyncInput syncInput = new StandardSyncInput()
           .withNamespaceDefinition(config.getNamespaceDefinition())
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java
index cc716a3e8802..e0ca2929904a 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java
@@ -145,7 +145,7 @@ private CheckedSupplier, Exception> getLegacyWork
             processFactory, new DefaultNormalizationRunner(
                 processFactory,
                 destinationLauncherConfig.getNormalizationDockerImage(),
-                destinationLauncherConfig.getIntegrationType())));
+                destinationLauncherConfig.getNormalizationIntegrationType())));
   }
 
   private CheckedSupplier, Exception> getContainerLauncherWorkerFactory(
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java
index 260644334be4..a498162b8eef 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java
@@ -153,7 +153,7 @@ private CheckedSupplier, Except
         new DefaultNormalizationRunner(
             processFactory,
             destinationLauncherConfig.getNormalizationDockerImage(),
-            destinationLauncherConfig.getIntegrationType()),
+            destinationLauncherConfig.getNormalizationIntegrationType()),
         workerEnvironment);
   }
 

From 85ce1f5bcf640bbec584ae3ac69b947dec826265 Mon Sep 17 00:00:00 2001
From: AndriiKorotkov 
Date: Mon, 12 Dec 2022 14:26:44 +0200
Subject: [PATCH 26/34] renamed extra dependencies

---
 .../bases/standard-destination-test/build.gradle                 | 1 -
 1 file changed, 1 deletion(-)

diff --git a/airbyte-integrations/bases/standard-destination-test/build.gradle b/airbyte-integrations/bases/standard-destination-test/build.gradle
index 7b73aa3d1da7..bb98faae7b65 100644
--- a/airbyte-integrations/bases/standard-destination-test/build.gradle
+++ b/airbyte-integrations/bases/standard-destination-test/build.gradle
@@ -5,7 +5,6 @@ dependencies {
     implementation project(':airbyte-db:db-lib')
     implementation project(':airbyte-commons-worker')
     implementation project(':airbyte-config:config-models')
-    implementation project(':airbyte-config:config-persistence')
     implementation project(':airbyte-config:init')
     implementation project(':airbyte-json-validation')
     implementation project(':airbyte-integrations:bases:base-java')

From 1dff83571e044308fd244b7187c44c6159f739fc Mon Sep 17 00:00:00 2001
From: AndriiKorotkov 
Date: Mon, 12 Dec 2022 14:45:16 +0200
Subject: [PATCH 27/34] updated docs

---
 .../normalized_tables_schema.txt              | 42 +++++++++----------
 .../configs_database/schema_dump.txt          |  2 +-
 2 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/airbyte-db/db-lib/src/main/resources/configs_database/normalized_tables_schema.txt b/airbyte-db/db-lib/src/main/resources/configs_database/normalized_tables_schema.txt
index c4b9a68f331a..82533ef0e397 100644
--- a/airbyte-db/db-lib/src/main/resources/configs_database/normalized_tables_schema.txt
+++ b/airbyte-db/db-lib/src/main/resources/configs_database/normalized_tables_schema.txt
@@ -71,27 +71,27 @@ Referenced by:
 
 
                                Table "public.actor_definition"
-        Column              |           Type           | Collation | Nullable |      Default
-----------------------------+--------------------------+-----------+----------+-------------------
- id                         | uuid                     |           | not null |
- name                       | character varying(256)   |           | not null |
- docker_repository          | character varying(256)   |           | not null |
- docker_image_tag           | character varying(256)   |           | not null |
- documentation_url          | character varying(256)   |           |          |
- icon                       | character varying(256)   |           |          |
- actor_type                 | actor_type               |           | not null |
- source_type                | source_type              |           |          |
- spec                       | jsonb                    |           | not null |
- created_at                 | timestamp with time zone |           | not null | CURRENT_TIMESTAMP
- updated_at                 | timestamp with time zone |           | not null | CURRENT_TIMESTAMP
- tombstone                  | boolean                  |           | not null | false
- release_stage              | release_stage            |           |          |
- release_date               | date                     |           |          |
- resource_requirements      | jsonb                    |           |          |
- normalization_repository   | character varying(255)   |           |          |
- normalization_tag          | character varying(255)   |           |          |
- supports_dbt               | boolean                  |           |          |
- integration_type           | character varying(255)   |           |          |
+        Column                  |           Type           | Collation | Nullable |      Default
+--------------------------------+--------------------------+-----------+----------+-------------------
+ id                             | uuid                     |           | not null |
+ name                           | character varying(256)   |           | not null |
+ docker_repository              | character varying(256)   |           | not null |
+ docker_image_tag               | character varying(256)   |           | not null |
+ documentation_url              | character varying(256)   |           |          |
+ icon                           | character varying(256)   |           |          |
+ actor_type                     | actor_type               |           | not null |
+ source_type                    | source_type              |           |          |
+ spec                           | jsonb                    |           | not null |
+ created_at                     | timestamp with time zone |           | not null | CURRENT_TIMESTAMP
+ updated_at                     | timestamp with time zone |           | not null | CURRENT_TIMESTAMP
+ tombstone                      | boolean                  |           | not null | false
+ release_stage                  | release_stage            |           |          |
+ release_date                   | date                     |           |          |
+ resource_requirements          | jsonb                    |           |          |
+ normalization_repository       | character varying(255)   |           |          |
+ normalization_tag              | character varying(255)   |           |          |
+ supports_dbt                   | boolean                  |           |          |
+ normalization_integration_type | character varying(255)   |           |          |
 Indexes:
     "actor_definition_pkey" PRIMARY KEY, btree (id)
 Referenced by:
diff --git a/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt b/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt
index 473d5999b054..4e3b80eae471 100644
--- a/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt
+++ b/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt
@@ -57,7 +57,7 @@ create table "public"."actor_definition"(
   "normalization_repository" varchar(255) null,
   "normalization_tag" varchar(255) null,
   "supports_dbt" bool null,
-  "integration_type" varchar(255) null,
+  "normalization_integration_type" varchar(255) null,
   constraint "actor_definition_pkey"
     primary key ("id")
 );

From 8aa5effb602f0ef2e62c0f0c49c319632c5104cb Mon Sep 17 00:00:00 2001
From: AndriiKorotkov 
Date: Tue, 13 Dec 2022 11:08:32 +0200
Subject: [PATCH 28/34] updated docs

---
 .../scheduling/activities/GenerateInputActivityImpl.java     | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java
index 0b5a72eb25be..88f274ebf947 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java
@@ -31,6 +31,7 @@
 import jakarta.inject.Singleton;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 
 @Singleton
@@ -97,7 +98,9 @@ public GeneratedJobInput getSyncWorkflowInput(final SyncInput input) {
           .findFirst();
       final String destinationNormalizationDockerImage = optionalDestinationDefinition.map(standardDestinationDefinition -> String.format("%s:%s",
           standardDestinationDefinition.getNormalizationRepository(), standardDestinationDefinition.getNormalizationTag())).orElse(null);
-      final boolean supportstDbt = optionalDestinationDefinition.isPresent() ? optionalDestinationDefinition.get().getSupportsDbt() : false;
+      final boolean supportstDbt = optionalDestinationDefinition.isPresent() && Objects.nonNull(optionalDestinationDefinition.get().getSupportsDbt())
+          ? optionalDestinationDefinition.get().getSupportsDbt()
+          : false;
       final String normalizationIntegrationType =
           optionalDestinationDefinition.map(StandardDestinationDefinition::getNormalizationIntegrationType).orElse(null);
 

From f196bea82ac8bedec9f0316bdd0df3bc19251bdc Mon Sep 17 00:00:00 2001
From: AndriiKorotkov 
Date: Wed, 14 Dec 2022 09:54:45 +0200
Subject: [PATCH 29/34] fixed minor remarks

---
 .../workers/normalization/DefaultNormalizationRunnerTest.java | 2 +-
 .../main/resources/types/StandardDestinationDefinition.yaml   | 4 ++--
 ...002__AddNormalizationIntegrationTypeToActorDefinition.java | 1 -
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
index 2959d4bfe823..0c3eb0633c06 100644
--- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
+++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/normalization/DefaultNormalizationRunnerTest.java
@@ -50,7 +50,7 @@ class DefaultNormalizationRunnerTest {
   private static final int JOB_ATTEMPT = 0;
 
   private static final String NORMALIZATION_IMAGE = "airbyte/normalization";
-  private static final String NORMALIZATION_TAG = "0.2.24";
+  private static final String NORMALIZATION_TAG = "42.42.42";
   private static final String INTEGRATION_TYPE = "postgres";
 
   private static Path logJobRoot;
diff --git a/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml b/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml
index 309267aa93d7..2c0ac7191bfb 100644
--- a/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml
+++ b/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml
@@ -63,10 +63,10 @@ properties:
     description: an optional field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.
   normalizationTag:
     type: string
-    description: an optional field indicating the tag of the repository to be used for normalization.
+    description: an optional field indicating the tag of the docker repository to be used for normalization.
   supportsDbt:
     type: boolean
     description: an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used.
   normalizationIntegrationType:
     type: string
-    description: an optional field indicating the type of integration dialect to use for normalization. If the value is NULL, the default dialect will be used.
+    description: an optional field indicating the type of integration dialect to use for normalization.
diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_23_002__AddNormalizationIntegrationTypeToActorDefinition.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_23_002__AddNormalizationIntegrationTypeToActorDefinition.java
index c1cb62d2f4c8..1f8f1726c897 100644
--- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_23_002__AddNormalizationIntegrationTypeToActorDefinition.java
+++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_40_23_002__AddNormalizationIntegrationTypeToActorDefinition.java
@@ -12,7 +12,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-// TODO: update migration description in the class name
 public class V0_40_23_002__AddNormalizationIntegrationTypeToActorDefinition extends BaseJavaMigration {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(

From 1df7802d7fca52819d577cc485168d58c78ac40f Mon Sep 17 00:00:00 2001
From: AndriiKorotkov 
Date: Wed, 14 Dec 2022 13:17:52 +0200
Subject: [PATCH 30/34] added NormalizationDestinationDefinitionConfig.yaml for
 StandardDestinationDefinition.yaml and updated configuration

---
 ...malizationDestinationDefinitionConfig.yaml | 21 +++++++++++
 .../types/StandardDestinationDefinition.yaml  | 11 ++----
 .../persistence/ActorDefinitionMigrator.java  |  9 +++--
 .../config/persistence/ConfigWriter.java      | 35 +++++++++++++------
 .../config/persistence/DbConverter.java       | 35 ++++++-------------
 .../job/errorreporter/JobErrorReporter.java   |  6 ++--
 .../errorreporter/JobErrorReporterTest.java   | 14 ++++----
 .../DestinationDefinitionsHandler.java        |  4 +--
 .../activities/GenerateInputActivityImpl.java | 14 +++++---
 9 files changed, 83 insertions(+), 66 deletions(-)
 create mode 100644 airbyte-config/config-models/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml

diff --git a/airbyte-config/config-models/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml b/airbyte-config/config-models/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml
new file mode 100644
index 000000000000..1746c233a39d
--- /dev/null
+++ b/airbyte-config/config-models/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml
@@ -0,0 +1,21 @@
+---
+"$schema": http://json-schema.org/draft-07/schema#
+"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml
+title: NormalizationDestinationDefinitionConfig
+description: describes a normalization config for destination definition
+type: object
+required:
+  - normalizationRepository
+  - normalizationTag
+  - normalizationIntegrationType
+additionalProperties: true
+properties:
+  normalizationRepository:
+    type: string
+    description: an optional field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.
+  normalizationTag:
+    type: string
+    description: an optional field indicating the tag of the docker repository to be used for normalization.
+  normalizationIntegrationType:
+    type: string
+    description: an optional field indicating the type of integration dialect to use for normalization.
diff --git a/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml b/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml
index 2c0ac7191bfb..103530d33b50 100644
--- a/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml
+++ b/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml
@@ -58,15 +58,8 @@ properties:
   protocolVersion:
     type: string
     description: the Airbyte Protocol version supported by the connector
-  normalizationRepository:
-    type: string
-    description: an optional field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.
-  normalizationTag:
-    type: string
-    description: an optional field indicating the tag of the docker repository to be used for normalization.
+  normalizationConfig:
+    "$ref": NormalizationDestinationDefinitionConfig.yaml
   supportsDbt:
     type: boolean
     description: an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used.
-  normalizationIntegrationType:
-    type: string
-    description: an optional field indicating the type of integration dialect to use for normalization.
diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ActorDefinitionMigrator.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ActorDefinitionMigrator.java
index 51b37e9d7795..8f6bb6fbaf1b 100644
--- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ActorDefinitionMigrator.java
+++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ActorDefinitionMigrator.java
@@ -16,10 +16,7 @@
 import io.airbyte.commons.util.MoreIterators;
 import io.airbyte.commons.version.AirbyteProtocolVersion;
 import io.airbyte.commons.version.AirbyteVersion;
-import io.airbyte.config.AirbyteConfig;
-import io.airbyte.config.ConfigSchema;
-import io.airbyte.config.StandardDestinationDefinition;
-import io.airbyte.config.StandardSourceDefinition;
+import io.airbyte.config.*;
 import io.airbyte.config.StandardSourceDefinition.SourceType;
 import io.airbyte.db.ExceptionWrappingDatabase;
 import io.airbyte.db.instance.configs.jooq.generated.enums.ActorType;
@@ -271,7 +268,9 @@ private void writeOrUpdateStandardDefinition(final DSLContext ctx,
       sourceDef.withProtocolVersion(getProtocolVersion(sourceDef.getSpec()));
       ConfigWriter.writeStandardSourceDefinition(Collections.singletonList(sourceDef), ctx);
     } else if (configType == ConfigSchema.STANDARD_DESTINATION_DEFINITION) {
-      final StandardDestinationDefinition destDef = Jsons.object(definition, StandardDestinationDefinition.class);
+      final NormalizationDestinationDefinitionConfig normalizationConfig = Jsons.object(definition, NormalizationDestinationDefinitionConfig.class);
+      final StandardDestinationDefinition destDef = Jsons.object(definition, StandardDestinationDefinition.class)
+          .withNormalizationConfig(normalizationConfig);
       destDef.withProtocolVersion(getProtocolVersion(destDef.getSpec()));
       ConfigWriter.writeStandardDestinationDefinition(Collections.singletonList(destDef), ctx);
     } else {
diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java
index 45c61f0b2f33..00640eadfd9b 100644
--- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java
+++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java
@@ -19,11 +19,8 @@
 import io.airbyte.db.instance.configs.jooq.generated.enums.SourceType;
 import java.time.LocalDate;
 import java.time.OffsetDateTime;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.Map.Entry;
-import java.util.Set;
-import java.util.UUID;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.jooq.DSLContext;
@@ -175,10 +172,19 @@ static void writeStandardDestinationDefinition(final List metadata = MoreMaps.merge(
               commonMetadata,
-              getNormalizationMetadata(destinationDefinition.getNormalizationRepository()),
+              getNormalizationMetadata(destinationDefinition.getNormalizationConfig().getNormalizationRepository()),
               prefixConnectorMetadataKeys(getSourceMetadata(sourceDefinition), "source"),
               getDestinationMetadata(destinationDefinition));
-          final String dockerImage = DockerUtils.getTaggedImageName(destinationDefinition.getNormalizationRepository(),
-              destinationDefinition.getNormalizationTag());
+          final String dockerImage = DockerUtils.getTaggedImageName(destinationDefinition.getNormalizationConfig().getNormalizationRepository(),
+              destinationDefinition.getNormalizationConfig().getNormalizationTag());
 
           reportJobFailureReason(workspace, failureReason, dockerImage, metadata);
         }
diff --git a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/JobErrorReporterTest.java b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/JobErrorReporterTest.java
index 6f71d8f98aa4..8bd3c38f32a3 100644
--- a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/JobErrorReporterTest.java
+++ b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/JobErrorReporterTest.java
@@ -6,15 +6,10 @@
 
 import static org.mockito.Mockito.mock;
 
-import io.airbyte.config.AttemptFailureSummary;
+import io.airbyte.config.*;
 import io.airbyte.config.Configs.DeploymentMode;
-import io.airbyte.config.FailureReason;
 import io.airbyte.config.FailureReason.FailureOrigin;
 import io.airbyte.config.FailureReason.FailureType;
-import io.airbyte.config.Metadata;
-import io.airbyte.config.StandardDestinationDefinition;
-import io.airbyte.config.StandardSourceDefinition;
-import io.airbyte.config.StandardWorkspace;
 import io.airbyte.config.persistence.ConfigNotFoundException;
 import io.airbyte.config.persistence.ConfigRepository;
 import io.airbyte.persistence.job.WebUrlHelper;
@@ -39,6 +34,7 @@ class JobErrorReporterTest {
   private static final String AIRBYTE_VERSION = "0.1.40";
   private static final String NORMALIZATION_IMAGE = "airbyte/normalization";
   private static final String NORMALIZATION_VERSION = "0.2.24";
+  private static final String NORMALIZATION_INTEGRATION_TYPE = "snowflake";
   private static final UUID SOURCE_DEFINITION_ID = UUID.randomUUID();
   private static final String SOURCE_DEFINITION_NAME = "stripe";
   private static final String SOURCE_DOCKER_REPOSITORY = "airbyte/source-stripe";
@@ -138,8 +134,10 @@ void testReportSyncJobFailure() {
             .withDockerRepository(DESTINATION_DOCKER_REPOSITORY)
             .withReleaseStage(DESTINATION_RELEASE_STAGE)
             .withDestinationDefinitionId(DESTINATION_DEFINITION_ID)
-            .withNormalizationRepository(NORMALIZATION_IMAGE)
-            .withNormalizationTag(NORMALIZATION_VERSION)
+            .withNormalizationConfig(new NormalizationDestinationDefinitionConfig()
+                .withNormalizationTag(NORMALIZATION_VERSION)
+                .withNormalizationRepository(NORMALIZATION_IMAGE)
+                .withNormalizationIntegrationType(NORMALIZATION_INTEGRATION_TYPE))
             .withName(DESTINATION_DEFINITION_NAME));
 
     final StandardWorkspace mWorkspace = Mockito.mock(StandardWorkspace.class);
diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java
index 98ab0e07b5d8..4b07f3bda272 100644
--- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java
+++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java
@@ -251,10 +251,8 @@ public DestinationDefinitionRead updateDestinationDefinition(final DestinationDe
         .withName(currentDestination.getName())
         .withDocumentationUrl(currentDestination.getDocumentationUrl())
         .withIcon(currentDestination.getIcon())
-        .withNormalizationRepository(currentDestination.getNormalizationRepository())
-        .withNormalizationTag(currentDestination.getNormalizationTag())
+        .withNormalizationConfig(currentDestination.getNormalizationConfig())
         .withSupportsDbt(currentDestination.getSupportsDbt())
-        .withNormalizationIntegrationType(currentDestination.getNormalizationIntegrationType())
         .withSpec(spec)
         .withProtocolVersion(airbyteProtocolVersion.serialize())
         .withTombstone(currentDestination.getTombstone())
diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java
index 88f274ebf947..2a550a62eb36 100644
--- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java
+++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java
@@ -96,13 +96,19 @@ public GeneratedJobInput getSyncWorkflowInput(final SyncInput input) {
               .equalsIgnoreCase(
                   DockerUtils.getTaggedImageName(destinationDefinition.getDockerRepository(), destinationDefinition.getDockerImageTag())))
           .findFirst();
-      final String destinationNormalizationDockerImage = optionalDestinationDefinition.map(standardDestinationDefinition -> String.format("%s:%s",
-          standardDestinationDefinition.getNormalizationRepository(), standardDestinationDefinition.getNormalizationTag())).orElse(null);
+      final String destinationNormalizationDockerImage = optionalDestinationDefinition
+          .filter(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getNormalizationConfig()))
+          .map(standardDestinationDefinition -> String.format("%s:%s",
+              standardDestinationDefinition.getNormalizationConfig().getNormalizationRepository(),
+              standardDestinationDefinition.getNormalizationConfig().getNormalizationTag()))
+          .orElse(null);
       final boolean supportstDbt = optionalDestinationDefinition.isPresent() && Objects.nonNull(optionalDestinationDefinition.get().getSupportsDbt())
           ? optionalDestinationDefinition.get().getSupportsDbt()
           : false;
-      final String normalizationIntegrationType =
-          optionalDestinationDefinition.map(StandardDestinationDefinition::getNormalizationIntegrationType).orElse(null);
+      final String normalizationIntegrationType = optionalDestinationDefinition
+          .filter(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getNormalizationConfig()))
+          .map(standardDestinationDefinition -> standardDestinationDefinition.getNormalizationConfig().getNormalizationIntegrationType())
+          .orElse(null);
 
       final IntegrationLauncherConfig sourceLauncherConfig = new IntegrationLauncherConfig()
           .withJobId(String.valueOf(jobId))

From 33fba46b2b8c72c2f56a27eaa2a6276f605c6ac9 Mon Sep 17 00:00:00 2001
From: AndriiKorotkov 
Date: Wed, 14 Dec 2022 13:21:35 +0200
Subject: [PATCH 31/34] updated normalization tag

---
 .../seed/destination_definitions.yaml         | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
index 06c8d17f581d..2bef21c681ae 100644
--- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
+++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
@@ -43,7 +43,7 @@
   documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery
   icon: bigquery.svg
   normalizationRepository: airbyte/normalization
-  normalizationTag: 0.2.24
+  normalizationTag: 0.2.25
   supportsDbt: true
   normalizationIntegrationType: bigquery
   resourceRequirements:
@@ -60,7 +60,7 @@
   documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery
   icon: bigquery.svg
   normalizationRepository: airbyte/normalization
-  normalizationTag: 0.2.24
+  normalizationTag: 0.2.25
   normalizationIntegrationType: bigquery
   resourceRequirements:
     jobSpecific:
@@ -91,7 +91,7 @@
   icon: clickhouse.svg
   releaseStage: alpha
   normalizationRepository: airbyte/normalization-clickhouse
-  normalizationTag: 0.2.24
+  normalizationTag: 0.2.25
   supportsDbt: true
   normalizationIntegrationType: clickhouse
 - name: Cloudflare R2
@@ -204,7 +204,7 @@
   icon: mssql.svg
   releaseStage: alpha
   normalizationRepository: airbyte/normalization-mssql
-  normalizationTag: 0.2.24
+  normalizationTag: 0.2.25
   supportsDbt: true
   normalizationIntegrationType: mssql
 - name: MeiliSearch
@@ -229,7 +229,7 @@
   icon: mysql.svg
   releaseStage: alpha
   normalizationRepository: airbyte/normalization-mysql
-  normalizationTag: 0.2.24
+  normalizationTag: 0.2.25
   supportsDbt: true
   normalizationIntegrationType: mysql
 - name: Oracle
@@ -240,7 +240,7 @@
   icon: oracle.svg
   releaseStage: alpha
   normalizationRepository: airbyte/normalization-oracle
-  normalizationTag: 0.2.24
+  normalizationTag: 0.2.25
   supportsDbt: true
   normalizationIntegrationType: oracle
 - name: Postgres
@@ -251,7 +251,7 @@
   icon: postgresql.svg
   releaseStage: alpha
   normalizationRepository: airbyte/normalization
-  normalizationTag: 0.2.24
+  normalizationTag: 0.2.25
   supportsDbt: true
   normalizationIntegrationType: postgres
 - name: Pulsar
@@ -282,7 +282,7 @@
   documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift
   icon: redshift.svg
   normalizationRepository: airbyte/normalization-redshift
-  normalizationTag: 0.2.24
+  normalizationTag: 0.2.25
   supportsDbt: true
   normalizationIntegrationType: redshift
   resourceRequirements:
@@ -337,7 +337,7 @@
   documentationUrl: https://docs.airbyte.com/integrations/destinations/snowflake
   icon: snowflake.svg
   normalizationRepository: airbyte/normalization-snowflake
-  normalizationTag: 0.2.24
+  normalizationTag: 0.2.25
   supportsDbt: true
   normalizationIntegrationType: snowflake
   resourceRequirements:
@@ -390,7 +390,7 @@
   icon: tidb.svg
   releaseStage: alpha
   normalizationRepository: airbyte/normalization-tidb
-  normalizationTag: 0.2.24
+  normalizationTag: 0.2.25
   supportsDbt: true
   normalizationIntegrationType: tidb
 - name: Typesense

From 3e0a6b3e4f82c9dc56ed4fbb76d670517b2b58e3 Mon Sep 17 00:00:00 2001
From: AndriiKorotkov 
Date: Wed, 14 Dec 2022 13:49:44 +0200
Subject: [PATCH 32/34] updated DestinationAcceptanceTest.java

---
 .../destination/DestinationAcceptanceTest.java            | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
index 6e07d54d60aa..ae25c0128f34 100644
--- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
+++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
@@ -141,7 +141,8 @@ private Optional getOptionalDestinationDefinition
 
   protected String getNormalizationImageName() {
     return getOptionalDestinationDefinitionFromProvider(getImageNameWithoutTag())
-        .map(standardDestinationDefinition -> standardDestinationDefinition.getNormalizationRepository() + ":"
+        .filter(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getNormalizationConfig()))
+        .map(standardDestinationDefinition -> standardDestinationDefinition.getNormalizationConfig().getNormalizationRepository() + ":"
             + NORMALIZATION_VERSION)
         .orElse(null);
   }
@@ -227,8 +228,9 @@ protected boolean implementsAppend() throws WorkerException {
 
   protected boolean normalizationFromDefinition() {
     return getOptionalDestinationDefinitionFromProvider(getImageNameWithoutTag())
-        .map(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getNormalizationRepository())
-            && Objects.nonNull(standardDestinationDefinition.getNormalizationTag()))
+        .filter(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getNormalizationConfig()))
+        .map(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getNormalizationConfig().getNormalizationRepository())
+            && Objects.nonNull(standardDestinationDefinition.getNormalizationConfig().getNormalizationTag()))
         .orElse(false);
   }
 

From 811e8558f7bbe169f47a60aee1b37be9c1531813 Mon Sep 17 00:00:00 2001
From: AndriiKorotkov 
Date: Wed, 14 Dec 2022 14:01:28 +0200
Subject: [PATCH 33/34] updated DestinationAcceptanceTest.java

---
 .../standardtest/destination/DestinationAcceptanceTest.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
index ae25c0128f34..04efcf05381a 100644
--- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
+++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
@@ -243,7 +243,8 @@ protected boolean dbtFromDefinition() {
 
   protected String getNormalizationIntegrationType() {
     return getOptionalDestinationDefinitionFromProvider(getImageNameWithoutTag())
-        .map(StandardDestinationDefinition::getNormalizationIntegrationType)
+        .filter(standardDestinationDefinition -> Objects.nonNull(standardDestinationDefinition.getNormalizationConfig()))
+        .map(standardDestinationDefinition -> standardDestinationDefinition.getNormalizationConfig().getNormalizationIntegrationType())
         .orElse(null);
   }
 

From 3a09ee0ef6c7910f1d76e979229bcbbe23797406 Mon Sep 17 00:00:00 2001
From: AndriiKorotkov 
Date: Thu, 15 Dec 2022 11:08:55 +0200
Subject: [PATCH 34/34] updated imports and descriptions

---
 ...malizationDestinationDefinitionConfig.yaml |  6 +--
 .../persistence/ActorDefinitionMigrator.java  |  6 ++-
 .../config/persistence/DbConverter.java       | 20 +++++++++-
 .../errorreporter/JobErrorReporterTest.java   |  8 +++-
 .../handlers/JobHistoryHandlerTest.java       | 37 +++++++++++++++++--
 5 files changed, 68 insertions(+), 9 deletions(-)

diff --git a/airbyte-config/config-models/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml b/airbyte-config/config-models/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml
index 1746c233a39d..69aacfa2dea8 100644
--- a/airbyte-config/config-models/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml
+++ b/airbyte-config/config-models/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml
@@ -12,10 +12,10 @@ additionalProperties: true
 properties:
   normalizationRepository:
     type: string
-    description: an optional field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.
+    description: a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.
   normalizationTag:
     type: string
-    description: an optional field indicating the tag of the docker repository to be used for normalization.
+    description: a field indicating the tag of the docker repository to be used for normalization.
   normalizationIntegrationType:
     type: string
-    description: an optional field indicating the type of integration dialect to use for normalization.
+    description: a field indicating the type of integration dialect to use for normalization.
diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ActorDefinitionMigrator.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ActorDefinitionMigrator.java
index 8f6bb6fbaf1b..dbcf9dc3985e 100644
--- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ActorDefinitionMigrator.java
+++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ActorDefinitionMigrator.java
@@ -16,7 +16,11 @@
 import io.airbyte.commons.util.MoreIterators;
 import io.airbyte.commons.version.AirbyteProtocolVersion;
 import io.airbyte.commons.version.AirbyteVersion;
-import io.airbyte.config.*;
+import io.airbyte.config.AirbyteConfig;
+import io.airbyte.config.ConfigSchema;
+import io.airbyte.config.NormalizationDestinationDefinitionConfig;
+import io.airbyte.config.StandardDestinationDefinition;
+import io.airbyte.config.StandardSourceDefinition;
 import io.airbyte.config.StandardSourceDefinition.SourceType;
 import io.airbyte.db.ExceptionWrappingDatabase;
 import io.airbyte.db.instance.configs.jooq.generated.enums.ActorType;
diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java
index ac5ccbc4bf45..e4b3fcb10031 100644
--- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java
+++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java
@@ -15,12 +15,30 @@
 
 import io.airbyte.commons.enums.Enums;
 import io.airbyte.commons.json.Jsons;
-import io.airbyte.config.*;
+import io.airbyte.config.ActorCatalog;
+import io.airbyte.config.ActorCatalogFetchEvent;
+import io.airbyte.config.ActorDefinitionResourceRequirements;
+import io.airbyte.config.DestinationConnection;
+import io.airbyte.config.DestinationOAuthParameter;
+import io.airbyte.config.FieldSelectionData;
+import io.airbyte.config.Geography;
 import io.airbyte.config.JobSyncConfig.NamespaceDefinitionType;
+import io.airbyte.config.NormalizationDestinationDefinitionConfig;
+import io.airbyte.config.Notification;
+import io.airbyte.config.ResourceRequirements;
+import io.airbyte.config.Schedule;
+import io.airbyte.config.ScheduleData;
+import io.airbyte.config.SourceConnection;
+import io.airbyte.config.SourceOAuthParameter;
+import io.airbyte.config.StandardDestinationDefinition;
+import io.airbyte.config.StandardSourceDefinition;
 import io.airbyte.config.StandardSourceDefinition.SourceType;
+import io.airbyte.config.StandardSync;
 import io.airbyte.config.StandardSync.NonBreakingChangesPreference;
 import io.airbyte.config.StandardSync.ScheduleType;
 import io.airbyte.config.StandardSync.Status;
+import io.airbyte.config.StandardWorkspace;
+import io.airbyte.config.WorkspaceServiceAccount;
 import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
 import io.airbyte.protocol.models.ConnectorSpecification;
 import java.time.LocalDateTime;
diff --git a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/JobErrorReporterTest.java b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/JobErrorReporterTest.java
index 8bd3c38f32a3..7f63364c3665 100644
--- a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/JobErrorReporterTest.java
+++ b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/JobErrorReporterTest.java
@@ -6,10 +6,16 @@
 
 import static org.mockito.Mockito.mock;
 
-import io.airbyte.config.*;
+import io.airbyte.config.AttemptFailureSummary;
 import io.airbyte.config.Configs.DeploymentMode;
+import io.airbyte.config.FailureReason;
 import io.airbyte.config.FailureReason.FailureOrigin;
 import io.airbyte.config.FailureReason.FailureType;
+import io.airbyte.config.Metadata;
+import io.airbyte.config.NormalizationDestinationDefinitionConfig;
+import io.airbyte.config.StandardDestinationDefinition;
+import io.airbyte.config.StandardSourceDefinition;
+import io.airbyte.config.StandardWorkspace;
 import io.airbyte.config.persistence.ConfigNotFoundException;
 import io.airbyte.config.persistence.ConfigRepository;
 import io.airbyte.persistence.job.WebUrlHelper;
diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/JobHistoryHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/JobHistoryHandlerTest.java
index d53044e95596..cb68ad2b8205 100644
--- a/airbyte-server/src/test/java/io/airbyte/server/handlers/JobHistoryHandlerTest.java
+++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/JobHistoryHandlerTest.java
@@ -10,12 +10,38 @@
 import static org.mockito.Mockito.when;
 
 import com.google.common.collect.ImmutableList;
-import io.airbyte.api.model.generated.*;
+import io.airbyte.api.model.generated.AttemptInfoRead;
+import io.airbyte.api.model.generated.AttemptNormalizationStatusRead;
+import io.airbyte.api.model.generated.AttemptNormalizationStatusReadList;
+import io.airbyte.api.model.generated.AttemptRead;
+import io.airbyte.api.model.generated.ConnectionRead;
+import io.airbyte.api.model.generated.DestinationIdRequestBody;
+import io.airbyte.api.model.generated.DestinationRead;
+import io.airbyte.api.model.generated.JobConfigType;
+import io.airbyte.api.model.generated.JobDebugInfoRead;
+import io.airbyte.api.model.generated.JobDebugRead;
+import io.airbyte.api.model.generated.JobIdRequestBody;
+import io.airbyte.api.model.generated.JobInfoLightRead;
+import io.airbyte.api.model.generated.JobInfoRead;
+import io.airbyte.api.model.generated.JobListRequestBody;
+import io.airbyte.api.model.generated.JobRead;
+import io.airbyte.api.model.generated.JobReadList;
+import io.airbyte.api.model.generated.JobWithAttemptsRead;
+import io.airbyte.api.model.generated.LogRead;
+import io.airbyte.api.model.generated.Pagination;
+import io.airbyte.api.model.generated.SourceIdRequestBody;
+import io.airbyte.api.model.generated.SourceRead;
 import io.airbyte.commons.enums.Enums;
 import io.airbyte.commons.version.AirbyteVersion;
-import io.airbyte.config.*;
 import io.airbyte.config.Configs.WorkerEnvironment;
+import io.airbyte.config.DestinationConnection;
+import io.airbyte.config.JobCheckConnectionConfig;
+import io.airbyte.config.JobConfig;
 import io.airbyte.config.JobConfig.ConfigType;
+import io.airbyte.config.SourceConnection;
+import io.airbyte.config.StandardDestinationDefinition;
+import io.airbyte.config.StandardSourceDefinition;
+import io.airbyte.config.StandardSync;
 import io.airbyte.config.helpers.LogConfigs;
 import io.airbyte.config.persistence.ConfigNotFoundException;
 import io.airbyte.persistence.job.JobPersistence;
@@ -34,7 +60,12 @@
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.nio.file.Path;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import org.junit.jupiter.api.BeforeEach;