From 4d1a788177902872a96eabd05db9c465688fa7fc Mon Sep 17 00:00:00 2001 From: Charles Date: Thu, 22 Oct 2020 20:06:33 -0700 Subject: [PATCH] standard source tests (#686) --- .../build.gradle | 0 .../integrations/base/TestDestination.java | 0 .../main/resources/exchange_rate_catalog.json | 0 .../main/resources/exchange_rate_messages.txt | 0 .../src/main/resources/stripe_messages.txt | 0 .../bases/standard-source-test/.dockerignore | 4 + .../bases/standard-source-test/Dockerfile | 30 ++ .../bases/standard-source-test/build.gradle | 23 ++ .../bases/standard-source-test/entrypoint.sh | 5 + .../base/ExecutableTestSource.java | 104 +++++++ .../airbyte/integrations/base/TestSource.java | 276 ++++++++++++++++++ .../integrations/base/TestSourceMain.java | 110 +++++++ .../destination-bigquery/build.gradle | 2 +- .../connectors/destination-csv/build.gradle | 2 +- .../destination-postgres/build.gradle | 2 +- .../source-github-singer/build.gradle | 10 + .../standard_test/catalog.json | 169 +++++++++++ .../gradle/commons/integrations/image.gradle | 2 + .../integrations/standard-source-test.gradle | 38 +++ 19 files changed, 774 insertions(+), 3 deletions(-) rename airbyte-integrations/bases/{integration-test-lib => destination-test-lib}/build.gradle (100%) rename airbyte-integrations/bases/{integration-test-lib => destination-test-lib}/src/main/java/io/airbyte/integrations/base/TestDestination.java (100%) rename airbyte-integrations/bases/{integration-test-lib => destination-test-lib}/src/main/resources/exchange_rate_catalog.json (100%) rename airbyte-integrations/bases/{integration-test-lib => destination-test-lib}/src/main/resources/exchange_rate_messages.txt (100%) rename airbyte-integrations/bases/{integration-test-lib => destination-test-lib}/src/main/resources/stripe_messages.txt (100%) create mode 100644 airbyte-integrations/bases/standard-source-test/.dockerignore create mode 100644 airbyte-integrations/bases/standard-source-test/Dockerfile create mode 100644 airbyte-integrations/bases/standard-source-test/build.gradle create mode 100755 airbyte-integrations/bases/standard-source-test/entrypoint.sh create mode 100644 airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/base/ExecutableTestSource.java create mode 100644 airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/base/TestSource.java create mode 100644 airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/base/TestSourceMain.java create mode 100644 airbyte-integrations/connectors/source-github-singer/standard_test/catalog.json create mode 100644 tools/gradle/commons/integrations/standard-source-test.gradle diff --git a/airbyte-integrations/bases/integration-test-lib/build.gradle b/airbyte-integrations/bases/destination-test-lib/build.gradle similarity index 100% rename from airbyte-integrations/bases/integration-test-lib/build.gradle rename to airbyte-integrations/bases/destination-test-lib/build.gradle diff --git a/airbyte-integrations/bases/integration-test-lib/src/main/java/io/airbyte/integrations/base/TestDestination.java b/airbyte-integrations/bases/destination-test-lib/src/main/java/io/airbyte/integrations/base/TestDestination.java similarity index 100% rename from airbyte-integrations/bases/integration-test-lib/src/main/java/io/airbyte/integrations/base/TestDestination.java rename to airbyte-integrations/bases/destination-test-lib/src/main/java/io/airbyte/integrations/base/TestDestination.java diff --git a/airbyte-integrations/bases/integration-test-lib/src/main/resources/exchange_rate_catalog.json b/airbyte-integrations/bases/destination-test-lib/src/main/resources/exchange_rate_catalog.json similarity index 100% rename from airbyte-integrations/bases/integration-test-lib/src/main/resources/exchange_rate_catalog.json rename to airbyte-integrations/bases/destination-test-lib/src/main/resources/exchange_rate_catalog.json diff --git a/airbyte-integrations/bases/integration-test-lib/src/main/resources/exchange_rate_messages.txt b/airbyte-integrations/bases/destination-test-lib/src/main/resources/exchange_rate_messages.txt similarity index 100% rename from airbyte-integrations/bases/integration-test-lib/src/main/resources/exchange_rate_messages.txt rename to airbyte-integrations/bases/destination-test-lib/src/main/resources/exchange_rate_messages.txt diff --git a/airbyte-integrations/bases/integration-test-lib/src/main/resources/stripe_messages.txt b/airbyte-integrations/bases/destination-test-lib/src/main/resources/stripe_messages.txt similarity index 100% rename from airbyte-integrations/bases/integration-test-lib/src/main/resources/stripe_messages.txt rename to airbyte-integrations/bases/destination-test-lib/src/main/resources/stripe_messages.txt diff --git a/airbyte-integrations/bases/standard-source-test/.dockerignore b/airbyte-integrations/bases/standard-source-test/.dockerignore new file mode 100644 index 000000000000..6145f27e93a0 --- /dev/null +++ b/airbyte-integrations/bases/standard-source-test/.dockerignore @@ -0,0 +1,4 @@ +* +!Dockerfile +!build +!entrypoint.sh diff --git a/airbyte-integrations/bases/standard-source-test/Dockerfile b/airbyte-integrations/bases/standard-source-test/Dockerfile new file mode 100644 index 000000000000..82f3eadafa6d --- /dev/null +++ b/airbyte-integrations/bases/standard-source-test/Dockerfile @@ -0,0 +1,30 @@ +FROM openjdk:14.0.2-slim + +# Install Docker to launch worker images. Eventually should be replaced with Docker-java. +# See https://gitter.im/docker-java/docker-java?at=5f3eb87ba8c1780176603f4e for more information on why we are not currently using Docker-java +RUN apt-get update && apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg-agent \ + software-properties-common +RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - +RUN add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/debian \ + $(lsb_release -cs) \ + stable" +RUN apt-get update && apt-get install -y docker-ce-cli jq + +ENV APPLICATION standard-source-test + +WORKDIR /app + +COPY entrypoint.sh . +COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar + +RUN tar xf ${APPLICATION}.tar --strip-components=1 + +ENTRYPOINT ["/app/entrypoint.sh"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/standard-source-test diff --git a/airbyte-integrations/bases/standard-source-test/build.gradle b/airbyte-integrations/bases/standard-source-test/build.gradle new file mode 100644 index 000000000000..78ef8ca04712 --- /dev/null +++ b/airbyte-integrations/bases/standard-source-test/build.gradle @@ -0,0 +1,23 @@ +plugins { + id 'application' +} + +apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') + +dependencies { + implementation project(':airbyte-config:models') + implementation project(':airbyte-protocol:models') + implementation project(':airbyte-workers') + + // todo (cgardens) - Using JUnit4 instead of five. See TestMain.java for more details. + implementation group: 'junit', name: 'junit', version: '4.12' + + + implementation 'net.sourceforge.argparse4j:argparse4j:0.8.1' +} + +application { + mainClass = 'io.airbyte.integrations.base.TestSourceMain' +} + +buildImage.dependsOn(assemble) diff --git a/airbyte-integrations/bases/standard-source-test/entrypoint.sh b/airbyte-integrations/bases/standard-source-test/entrypoint.sh new file mode 100755 index 000000000000..2032df7db986 --- /dev/null +++ b/airbyte-integrations/bases/standard-source-test/entrypoint.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -e + +/app/bin/${APPLICATION} "$@" diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/base/ExecutableTestSource.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/base/ExecutableTestSource.java new file mode 100644 index 000000000000..eee2c31846cb --- /dev/null +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/base/ExecutableTestSource.java @@ -0,0 +1,104 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.base; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import io.airbyte.protocol.models.AirbyteCatalog; +import io.airbyte.protocol.models.ConnectorSpecification; +import java.nio.file.Path; + +/** + * Extends TestSource such that it can be called using resources pulled from the file system. Will + * also add the ability to execute arbitrary scripts in the next version. + */ +public class ExecutableTestSource extends TestSource { + + public static class TestConfig { + + private final String imageName; + private final Path specPath; + private final Path configPath; + private final Path catalogPath; + + public TestConfig(String imageName, Path specPath, Path configPath, Path catalogPath) { + this.imageName = imageName; + this.specPath = specPath; + this.configPath = configPath; + this.catalogPath = catalogPath; + } + + public String getImageName() { + return imageName; + } + + public Path getSpecPath() { + return specPath; + } + + public Path getConfigPath() { + return configPath; + } + + public Path getCatalogPath() { + return catalogPath; + } + + } + + public static TestConfig TEST_CONFIG; + + @Override + protected ConnectorSpecification getSpec() { + return Jsons.deserialize(IOs.readFile(TEST_CONFIG.getSpecPath()), ConnectorSpecification.class); + } + + @Override + protected String getImageName() { + return TEST_CONFIG.getImageName(); + } + + @Override + protected JsonNode getConfig() { + return Jsons.deserialize(IOs.readFile(TEST_CONFIG.getConfigPath())); + } + + @Override + protected AirbyteCatalog getCatalog() { + return Jsons.deserialize(IOs.readFile(TEST_CONFIG.getCatalogPath()), AirbyteCatalog.class); + } + + @Override + protected void setup(TestDestinationEnv testEnv) throws Exception { + // no-op, for now + } + + @Override + protected void tearDown(TestDestinationEnv testEnv) throws Exception { + // no-op, for now + } + +} diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/base/TestSource.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/base/TestSource.java new file mode 100644 index 000000000000..a53e822d5d4b --- /dev/null +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/base/TestSource.java @@ -0,0 +1,276 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.base; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.config.JobGetSpecConfig; +import io.airbyte.config.StandardCheckConnectionInput; +import io.airbyte.config.StandardCheckConnectionOutput; +import io.airbyte.config.StandardCheckConnectionOutput.Status; +import io.airbyte.config.StandardDiscoverCatalogInput; +import io.airbyte.config.StandardDiscoverCatalogOutput; +import io.airbyte.config.StandardGetSpecOutput; +import io.airbyte.config.StandardSync.SyncMode; +import io.airbyte.config.StandardTapConfig; +import io.airbyte.protocol.models.AirbyteCatalog; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.AirbyteMessage.Type; +import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.workers.DefaultCheckConnectionWorker; +import io.airbyte.workers.DefaultDiscoverCatalogWorker; +import io.airbyte.workers.DefaultGetSpecWorker; +import io.airbyte.workers.OutputAndStatus; +import io.airbyte.workers.process.AirbyteIntegrationLauncher; +import io.airbyte.workers.process.DockerProcessBuilderFactory; +import io.airbyte.workers.process.ProcessBuilderFactory; +import io.airbyte.workers.protocols.airbyte.AirbyteSource; +import io.airbyte.workers.protocols.airbyte.DefaultAirbyteSource; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public abstract class TestSource { + + private TestDestinationEnv testEnv; + + private Path jobRoot; + protected Path localRoot; + private ProcessBuilderFactory pbf; + + /** + * Name of the docker image that the tests will run against. + * + * @return docker image name + */ + protected abstract String getImageName(); + + /** + * Specification for integration. Will be passed to integration where appropriate in each test. + * Should be valid. + * + * @return integration-specific configuration + */ + protected abstract ConnectorSpecification getSpec() throws Exception; + + /** + * Configuration specific to the integration. Will be passed to integration where appropriate in + * each test. Should be valid. + * + * @return integration-specific configuration + */ + protected abstract JsonNode getConfig() throws Exception; + + /** + * Catalog to be used when attempting read operations. + * + * @return the catalog + * @throws Exception - do what must be done. + */ + protected abstract AirbyteCatalog getCatalog() throws Exception; + + /** + * Function that performs any setup of external resources required for the test. e.g. instantiate a + * postgres database. This function will be called before EACH test. + * + * @param testEnv - information about the test environment. + * @throws Exception - can throw any exception, test framework will handle. + */ + protected abstract void setup(TestDestinationEnv testEnv) throws Exception; + + /** + * Function that performs any clean up of external resources required for the test. e.g. delete a + * postgres database. This function will be called after EACH test. It MUST remove all data in the + * destination so that there is no contamination across tests. + * + * @param testEnv - information about the test environment. + * @throws Exception - can throw any exception, test framework will handle. + */ + protected abstract void tearDown(TestDestinationEnv testEnv) throws Exception; + + @Before + public void setUpInternal() throws Exception { + Path testDir = Path.of("/tmp/airbyte_tests/"); + Files.createDirectories(testDir); + final Path workspaceRoot = Files.createTempDirectory(testDir, "test"); + jobRoot = Files.createDirectories(Path.of(workspaceRoot.toString(), "job")); + localRoot = Files.createTempDirectory(testDir, "output"); + testEnv = new TestDestinationEnv(localRoot); + + setup(testEnv); + + pbf = new DockerProcessBuilderFactory( + workspaceRoot, + workspaceRoot.toString(), + localRoot.toString(), + "host"); + } + + @After + public void tearDownInternal() throws Exception { + tearDown(testEnv); + } + + /** + * Verify that when the integrations returns a valid spec. + */ + @Test + public void testGetSpec() throws Exception { + final OutputAndStatus output = runSpec(); + assertTrue(output.getOutput().isPresent()); + assertEquals(getSpec(), output.getOutput().get().getSpecification()); + } + + /** + * Verify that when given valid credentials, that check connection returns a success response. + * Assume that the {@link TestSource#getConfig()} is valid. + */ + @Test + public void testCheckConnection() throws Exception { + final OutputAndStatus output = runCheck(); + assertTrue(output.getOutput().isPresent()); + assertEquals(Status.SUCCEEDED, output.getOutput().get().getStatus()); + } + + // /** + // * Verify that when given invalid credentials, that check connection returns a failed response. + // * Assume that the {@link TestSource#getFailCheckConfig()} is invalid. + // */ + // @Test + // public void testCheckConnectionInvalidCredentials() throws Exception { + // final OutputAndStatus output = runCheck(); + // assertTrue(output.getOutput().isPresent()); + // assertEquals(Status.FAILED, output.getOutput().get().getStatus()); + // } + + /** + * Verify that when given valid credentials, that discover returns a valid catalog. Assume that the + * {@link TestSource#getConfig()} is valid. + */ + @Test + public void testDiscover() throws Exception { + final OutputAndStatus output = runDiscover(); + assertTrue(output.getOutput().isPresent()); + // the worker validates that it is a valid catalog, so we do not need to validate again (as long as + // we use the worker, which we will not want to do long term). + assertNotNull(output.getOutput().get().getCatalog()); + assertEquals(getCatalog(), output.getOutput().get().getCatalog()); + } + + /** + * Verify that the integration successfully writes records. Tests a wide variety of messages and + * schemas (aspirationally, anyway). + */ + @Test + public void testRead() throws Exception { + final List recordMessages = runRead(getCatalog()).stream().filter(m -> m.getType() == Type.RECORD).collect(Collectors.toList()); + // the worker validates the messages, so we just validate the message, so we do not need to validate + // again (as long as we use the worker, which we will not want to do long term). + assertFalse(recordMessages.isEmpty()); + } + + /** + * Verify that the integration overwrites the first sync with the second sync. + */ + @Test + public void testSecondRead() throws Exception { + final List recordMessagesFirstRun = + runRead(getCatalog()).stream().filter(m -> m.getType() == Type.RECORD).collect(Collectors.toList()); + final List recordMessagesSecondRun = + runRead(getCatalog()).stream().filter(m -> m.getType() == Type.RECORD).collect(Collectors.toList()); + // the worker validates the messages, so we just validate the message, so we do not need to validate + // again (as long as we use the worker, which we will not want to do long term). + assertFalse(recordMessagesFirstRun.isEmpty()); + assertFalse(recordMessagesSecondRun.isEmpty()); + assertSameMessages(recordMessagesSecondRun, recordMessagesSecondRun); + } + + private OutputAndStatus runSpec() { + return new DefaultGetSpecWorker(new AirbyteIntegrationLauncher(getImageName(), pbf)) + .run(new JobGetSpecConfig().withDockerImage(getImageName()), jobRoot); + } + + private OutputAndStatus runCheck() throws Exception { + return new DefaultCheckConnectionWorker(new AirbyteIntegrationLauncher(getImageName(), pbf)) + .run(new StandardCheckConnectionInput().withConnectionConfiguration(getConfig()), jobRoot); + } + + private OutputAndStatus runDiscover() throws Exception { + return new DefaultDiscoverCatalogWorker(new AirbyteIntegrationLauncher(getImageName(), pbf)) + .run(new StandardDiscoverCatalogInput().withConnectionConfiguration(getConfig()), jobRoot); + } + + // todo (cgardens) - assume no state since we are all full refresh right now. + private List runRead(AirbyteCatalog catalog) throws Exception { + final StandardTapConfig tapConfig = new StandardTapConfig() + .withConnectionId(UUID.randomUUID()) + .withSourceConnectionConfiguration(getConfig()) + .withSyncMode(SyncMode.FULL_REFRESH) + .withCatalog(catalog); + + final AirbyteSource source = new DefaultAirbyteSource(new AirbyteIntegrationLauncher(getImageName(), pbf)); + final List messages = new ArrayList<>(); + + source.start(tapConfig, jobRoot); + while (!source.isFinished()) { + source.attemptRead().ifPresent(messages::add); + } + source.close(); + + return messages; + } + + private void assertSameMessages(List expected, List actual) { + // we want to ignore order in this comparison. + assertEquals(expected.size(), actual.size()); + assertTrue(expected.containsAll(actual)); + assertTrue(actual.containsAll(expected)); + } + + public static class TestDestinationEnv { + + private final Path localRoot; + + public TestDestinationEnv(Path localRoot) { + this.localRoot = localRoot; + } + + public Path getLocalRoot() { + return localRoot; + } + + } + +} diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/base/TestSourceMain.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/base/TestSourceMain.java new file mode 100644 index 000000000000..1fd295a67acf --- /dev/null +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/base/TestSourceMain.java @@ -0,0 +1,110 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.base; + +import io.airbyte.integrations.base.ExecutableTestSource.TestConfig; +import java.nio.file.Path; +import net.sourceforge.argparse4j.ArgumentParsers; +import net.sourceforge.argparse4j.inf.ArgumentParser; +import net.sourceforge.argparse4j.inf.ArgumentParserException; +import net.sourceforge.argparse4j.inf.Namespace; +import org.junit.internal.TextListener; +import org.junit.runner.JUnitCore; +import org.junit.runner.Result; + +/** + * Parse command line arguments and inject them into the test class before running the test. Then + * runs the tests. + */ +public class TestSourceMain { + + public static void main(String[] args) { + ArgumentParser parser = ArgumentParsers.newFor(TestSourceMain.class.getName()).build() + .defaultHelp(true) + .description("Run standard source tests"); + + parser.addArgument("--imageName") + .help("Name of the integration image"); + + parser.addArgument("--spec") + .help("Path to file that contains spec json"); + + parser.addArgument("--config") + .help("Path to file that contains config json"); + + parser.addArgument("--catalog") + .help("Path to file that contains catalog json"); + + Namespace ns = null; + try { + ns = parser.parseArgs(args); + } catch (ArgumentParserException e) { + parser.handleError(e); + System.exit(1); + } + + final String imageName = ns.getString("imageName"); + final String specFile = ns.getString("spec"); + final String configFile = ns.getString("config"); + final String catalogFile = ns.getString("catalog"); + ExecutableTestSource.TEST_CONFIG = new TestConfig(imageName, Path.of(specFile), Path.of(configFile), Path.of(catalogFile)); + + // todo (cgardens) - using JUnit4 (instead of 5) here for two reasons: 1) Cannot get JUnit5 to print + // output nearly as nicely as 4. 2) JUnit5 was running all tests twice. Likely there are workarounds + // to both, but I cut my losses and went for something that worked. + JUnitCore junit = new JUnitCore(); + junit.addListener(new TextListener(System.out)); + final Result result = junit.run(ExecutableTestSource.class); + + if (result.getFailureCount() > 0) { + System.exit(1); + } + + // this is how you you are supposed to do it in JUnit5 + // LauncherDiscoveryRequest request = LauncherDiscoveryRequestBuilder.request() + // .selectors( + // selectPackage("io.airbyte.integrations.base"), + // selectClass(TestBasicTest.class) + // selectClass(TestBasicTest.class, ExecutableTestSource.class) + // ) + // .filters(includeClassNamePatterns(".*Test")) + // .build(); + // + // TestPlan plan = LauncherFactory.create().discover(request); + // Launcher launcher = LauncherFactory.create(); + // + // + // + // // Register a listener of your choice + // SummaryGeneratingListener listener = new SummaryGeneratingListener(); + // launcher.registerTestExecutionListeners(listener); + // + // launcher.execute(plan, listener); + // + // listener.getSummary().printFailuresTo(new PrintWriter(System.out)); + // listener.getSummary().printTo(new PrintWriter(System.out)); + } + +} diff --git a/airbyte-integrations/connectors/destination-bigquery/build.gradle b/airbyte-integrations/connectors/destination-bigquery/build.gradle index 41aaf5f96aef..78ab5bbce830 100644 --- a/airbyte-integrations/connectors/destination-bigquery/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery/build.gradle @@ -14,7 +14,7 @@ dependencies { implementation project(':airbyte-protocol:models') implementation project(':airbyte-queue') - integrationTestImplementation project(':airbyte-integrations:bases:integration-test-lib') + integrationTestImplementation project(':airbyte-integrations:bases:destination-test-lib') } diff --git a/airbyte-integrations/connectors/destination-csv/build.gradle b/airbyte-integrations/connectors/destination-csv/build.gradle index 16a1ec6b6004..4d2b74dc5826 100644 --- a/airbyte-integrations/connectors/destination-csv/build.gradle +++ b/airbyte-integrations/connectors/destination-csv/build.gradle @@ -11,7 +11,7 @@ dependencies { implementation project(':airbyte-protocol:models') implementation project(':airbyte-integrations:bases:base-java') - integrationTestImplementation project(':airbyte-integrations:bases:integration-test-lib') + integrationTestImplementation project(':airbyte-integrations:bases:destination-test-lib') } application { diff --git a/airbyte-integrations/connectors/destination-postgres/build.gradle b/airbyte-integrations/connectors/destination-postgres/build.gradle index c4500fce95b0..5b4de2fac984 100644 --- a/airbyte-integrations/connectors/destination-postgres/build.gradle +++ b/airbyte-integrations/connectors/destination-postgres/build.gradle @@ -14,7 +14,7 @@ dependencies { testImplementation "org.testcontainers:postgresql:1.15.0-rc2" - integrationTestImplementation project(':airbyte-integrations:bases:integration-test-lib') + integrationTestImplementation project(':airbyte-integrations:bases:destination-test-lib') integrationTestImplementation "org.testcontainers:postgresql:1.15.0-rc2" } diff --git a/airbyte-integrations/connectors/source-github-singer/build.gradle b/airbyte-integrations/connectors/source-github-singer/build.gradle index 6ac705dce895..a1003dfca012 100644 --- a/airbyte-integrations/connectors/source-github-singer/build.gradle +++ b/airbyte-integrations/connectors/source-github-singer/build.gradle @@ -4,9 +4,19 @@ plugins { apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') +apply from: rootProject.file('tools/gradle/commons/integrations/standard-source-test.gradle') dependencies { } +standardSourceTest { + ext { + imageName = "${extractImageName(project.file('Dockerfile'))}:dev" + specPath = 'source_github_singer/spec.json' + configPath ='secrets/config.json' + catalogPath = 'standard_test/catalog.json' + } +} + buildImage.dependsOn ':airbyte-integrations:bases:base-singer:buildImage' integrationTest.dependsOn(buildImage) diff --git a/airbyte-integrations/connectors/source-github-singer/standard_test/catalog.json b/airbyte-integrations/connectors/source-github-singer/standard_test/catalog.json new file mode 100644 index 000000000000..9b38d3536c2e --- /dev/null +++ b/airbyte-integrations/connectors/source-github-singer/standard_test/catalog.json @@ -0,0 +1,169 @@ +{ + "streams": [ + { + "name": "commits", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "_sdc_repository": { + "type": ["string"] + }, + "sha": { + "type": ["null", "string"], + "description": "The git commit hash" + }, + "url": { + "type": ["null", "string"] + }, + "parents": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "sha": { + "type": ["null", "string"], + "description": "The git hash of the parent commit" + }, + "url": { + "type": ["null", "string"], + "description": "The URL to the parent commit" + }, + "html_url": { + "type": ["null", "string"], + "description": "The HTML URL to the parent commit" + } + } + } + }, + "files": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "filename": { + "type": ["null", "string"] + }, + "additions": { + "type": ["null", "number"] + }, + "deletions": { + "type": ["null", "number"] + }, + "changes": { + "type": ["null", "number"] + }, + "status": { + "type": ["null", "string"] + }, + "raw_url": { + "type": ["null", "string"] + }, + "blob_url": { + "type": ["null", "string"] + }, + "patch": { + "type": ["null", "string"] + } + } + } + }, + "html_url": { + "type": ["null", "string"], + "description": "The HTML URL to the commit" + }, + "comments_url": { + "type": ["null", "string"], + "description": "The URL to the commit's comments page" + }, + "commit": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "url": { + "type": ["null", "string"], + "description": "The URL to the commit" + }, + "tree": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "sha": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + }, + "author": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "date": { + "type": ["null", "string"], + "format": "date-time", + "description": "The date the author committed the change" + }, + "name": { + "type": ["null", "string"], + "description": "The author's name" + }, + "email": { + "type": ["null", "string"], + "description": "The author's email" + }, + "login": { + "type": ["null", "string"], + "description": "The author's login" + } + } + }, + "message": { + "type": ["null", "string"], + "description": "The commit message" + }, + "committer": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "date": { + "type": ["null", "string"], + "format": "date-time", + "description": "The date the committer committed the change" + }, + "name": { + "type": ["null", "string"], + "description": "The committer's name" + }, + "email": { + "type": ["null", "string"], + "description": "The committer's email" + }, + "login": { + "type": ["null", "string"], + "description": "The committer's login" + } + } + }, + "comment_count": { + "type": ["null", "integer"], + "description": "The number of comments on the commit" + } + } + }, + "pr_number": { + "type": ["null", "integer"] + }, + "pr_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + } + } + } + } + ] +} diff --git a/tools/gradle/commons/integrations/image.gradle b/tools/gradle/commons/integrations/image.gradle index c04cb1e2b528..a8321c98513e 100644 --- a/tools/gradle/commons/integrations/image.gradle +++ b/tools/gradle/commons/integrations/image.gradle @@ -21,3 +21,5 @@ task buildImage(type: DockerBuildImage) { // Docker will still cache the artifact, so it shouldn't be much slower. outputs.upToDateWhen { false } } + +buildImage.dependsOn(build) diff --git a/tools/gradle/commons/integrations/standard-source-test.gradle b/tools/gradle/commons/integrations/standard-source-test.gradle new file mode 100644 index 000000000000..867c83d7fbff --- /dev/null +++ b/tools/gradle/commons/integrations/standard-source-test.gradle @@ -0,0 +1,38 @@ +apply from: rootProject.file('tools/gradle/commons/docker.gradle') + +task standardSourceTest { + ext { + imageName = '' + specPath = '' + configPath = '' + catalogPath = '' + } + + doFirst { + exec { + println('standard test inputs') + println("imageName: ${imageName}") + println("specPath: ${specPath}") + println("configPath: ${configPath}") + println("catalogPath: ${catalogPath}") + workingDir rootDir + commandLine 'docker', 'run', '--rm', '-i', + // so that it has access to docker + '-v', "/var/run/docker.sock:/var/run/docker.sock", + // when launching the container within a container, it mounts the directory from + // the host filesystem, not the parent container. this forces /tmp to be the + // same directory for host, parent container, and child container. + '-v', "/tmp:/tmp", + // mount the project dir. all provided input paths must be relative to that dir. + '-v', "${project.projectDir.absolutePath}:/test_input", + '--name', "std-test-${project.name}", 'airbyte/standard-source-test:dev', + '--imageName', imageName, + '--spec', "/test_input/${specPath}", + '--config', "/test_input/${configPath}", + '--catalog', "/test_input/${catalogPath}" + } + } +} +standardSourceTest.dependsOn(':airbyte-integrations:bases:standard-source-test:buildImage') +standardSourceTest.dependsOn(compileJava) +standardSourceTest.dependsOn(buildImage)