diff --git a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/SeedConnectorSpecGenerator.java b/airbyte-config/specs/src/main/java/io/airbyte/config/specs/SeedConnectorSpecGenerator.java index 723464f2f308..c68ea8d1f99c 100644 --- a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/SeedConnectorSpecGenerator.java +++ b/airbyte-config/specs/src/main/java/io/airbyte/config/specs/SeedConnectorSpecGenerator.java @@ -14,9 +14,11 @@ import io.airbyte.commons.yaml.Yamls; import io.airbyte.config.DockerImageSpec; import io.airbyte.config.EnvConfigs; +import io.airbyte.config.Spec; import io.airbyte.protocol.models.ConnectorSpecification; import java.io.IOException; import java.nio.file.Path; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -27,21 +29,17 @@ import org.slf4j.LoggerFactory; /** - * This script is responsible for ensuring that up-to-date {@link ConnectorSpecification}s for every - * connector definition in the seed are stored in a corresponding resource file, for the purpose of - * seeding the specs into the config database on server startup. See - * ./airbyte-config/specs/readme.md for more details on how this class is run and how it fits into - * the project. + * This script is responsible for ensuring that up-to-date {@link ConnectorSpecification}s for every connector definition in the seed are stored in a + * corresponding resource file, for the purpose of seeding the specs into the config database on server startup. See ./airbyte-config/specs/readme.md + * for more details on how this class is run and how it fits into the project. *

- * Specs are stored in a separate file from the definitions in an effort to keep the definitions - * yaml files human-readable and easily-editable, as specs can be rather large. + * Specs are stored in a separate file from the definitions in an effort to keep the definitions yaml files human-readable and easily-editable, as + * specs can be rather large. *

- * Specs are fetched from the GCS spec cache bucket, so if any specs are missing from the bucket - * then this will fail. Note that this script only pulls specs from the bucket cache; it never - * pushes specs to the bucket. Since this script runs at build time, the decision was to depend on - * the bucket cache rather than running a docker container to fetch the spec during the build which - * could be slow and unwieldy. If there is a failure, check the bucket cache and figure out how to - * get the correct spec in there. + * Specs are fetched from the GCS spec cache bucket, so if any specs are missing from the bucket then this will fail. Note that this script only pulls + * specs from the bucket cache; it never pushes specs to the bucket. Since this script runs at build time, the decision was to depend on the bucket + * cache rather than running a docker container to fetch the spec during the build which could be slow and unwieldy. If there is a failure, check the + * bucket cache and figure out how to get the correct spec in there. */ @SuppressWarnings("PMD.SignatureDeclareThrowsException") public class SeedConnectorSpecGenerator { @@ -50,6 +48,8 @@ public class SeedConnectorSpecGenerator { private static final String DOCKER_IMAGE_TAG_FIELD = "dockerImageTag"; private static final String DOCKER_IMAGE_FIELD = "dockerImage"; private static final String SPEC_FIELD = "spec"; + private static final String SPECS_FIELD = "specs"; + private static final String SPEC_TYPE_FIELD = "spec_type"; private static final String SPEC_BUCKET_NAME = new EnvConfigs().getSpecCacheBucket(); private static final Logger LOGGER = LoggerFactory.getLogger(SeedConnectorSpecGenerator.class); @@ -95,19 +95,32 @@ private JsonNode yamlToJson(final Path root, final String fileName) { return Yamls.deserialize(yamlString); } + final List getSpecVariants(final JsonNode specs) { + if (specs == null) { + return Collections.emptyList(); + } + return MoreIterators.toList(specs.elements()) + .stream() + .map(json -> new Spec() + .withSpecType(Jsons.object(json.get(SPEC_TYPE_FIELD), Spec.SpecType.class)) + .withSpec(Jsons.object(json.get(SPEC_FIELD), ConnectorSpecification.class))) + .toList(); + } + @VisibleForTesting final List fetchUpdatedSeedSpecs(final JsonNode seedDefinitions, final JsonNode currentSeedSpecs) { final List seedDefinitionsDockerImages = MoreIterators.toList(seedDefinitions.elements()) .stream() .map(json -> String.format("%s:%s", json.get(DOCKER_REPOSITORY_FIELD).asText(), json.get(DOCKER_IMAGE_TAG_FIELD).asText())) - .collect(Collectors.toList()); + .toList(); final Map currentSeedImageToSpec = MoreIterators.toList(currentSeedSpecs.elements()) .stream() .collect(Collectors.toMap( json -> json.get(DOCKER_IMAGE_FIELD).asText(), json -> new DockerImageSpec().withDockerImage(json.get(DOCKER_IMAGE_FIELD).asText()) - .withSpec(Jsons.object(json.get(SPEC_FIELD), ConnectorSpecification.class)))); + .withSpec(Jsons.object(json.get(SPEC_FIELD), ConnectorSpecification.class)) + .withSpecs(getSpecVariants(json.get(SPECS_FIELD))))); return seedDefinitionsDockerImages .stream() diff --git a/airbyte-config/specs/src/test/java/io/airbyte/config/specs/SeedConnectorSpecGeneratorTest.java b/airbyte-config/specs/src/test/java/io/airbyte/config/specs/SeedConnectorSpecGeneratorTest.java index 5750a152487c..6b3facba568e 100644 --- a/airbyte-config/specs/src/test/java/io/airbyte/config/specs/SeedConnectorSpecGeneratorTest.java +++ b/airbyte-config/specs/src/test/java/io/airbyte/config/specs/SeedConnectorSpecGeneratorTest.java @@ -15,6 +15,8 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.config.DockerImageSpec; +import io.airbyte.config.Spec; +import io.airbyte.config.Spec.SpecType; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.protocol.models.ConnectorSpecification; import java.util.Arrays; @@ -138,8 +140,20 @@ void testNoFetchIsPerformedIfAllSpecsUpToDate() { .withDockerImageTag(DOCKER_TAG1) .withName(CONNECTOR_NAME1) .withDocumentationUrl(DOCUMENTATION_URL); - final ConnectorSpecification spec = new ConnectorSpecification().withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of("foo", "bar"))); - final DockerImageSpec dockerImageSpec = new DockerImageSpec().withDockerImage(DOCKER_REPOSITORY1 + ":" + DOCKER_TAG1).withSpec(spec); + final ConnectorSpecification defaultSpec = new ConnectorSpecification() + .withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of("foo", "bar"))); + final Spec cloudSpec = new Spec() + .withSpecType(SpecType.CLOUD) + .withSpec(new ConnectorSpecification() + .withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of("foo", "bar", "mode", "cloud")))); + final Spec ossSpec = new Spec() + .withSpecType(SpecType.CLOUD) + .withSpec(new ConnectorSpecification() + .withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of("foo", "bar", "mode", "oss")))); + + final DockerImageSpec dockerImageSpec = new DockerImageSpec() + .withDockerImage(DOCKER_REPOSITORY1 + ":" + DOCKER_TAG1).withSpec(defaultSpec) + .withSpecs(List.of(cloudSpec, ossSpec)); final JsonNode seedDefinitions = Jsons.jsonNode(List.of(sourceDefinition)); final JsonNode seedSpecs = Jsons.jsonNode(List.of(dockerImageSpec)); @@ -151,4 +165,19 @@ void testNoFetchIsPerformedIfAllSpecsUpToDate() { verify(bucketSpecFetcherMock, never()).attemptFetch(any()); } + @Test + void testGetSpecVariants() { + final Spec cloudSpec = new Spec() + .withSpecType(SpecType.CLOUD) + .withSpec(new ConnectorSpecification() + .withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of("foo", "bar", "mode", "cloud")))); + final Spec ossSpec = new Spec() + .withSpecType(SpecType.CLOUD) + .withSpec(new ConnectorSpecification() + .withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of("foo", "bar", "mode", "oss")))); + + assertEquals(List.of(cloudSpec, ossSpec), + seedConnectorSpecGenerator.getSpecVariants(Jsons.jsonNode(List.of(cloudSpec, ossSpec)))); + } + }