From a38dccb769a357d1f31afafacddc4a18a2804bc2 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Sat, 9 Oct 2021 20:36:43 -0700 Subject: [PATCH 01/17] Implement destination null --- .../connectors/destination-null/.dockerignore | 3 + .../connectors/destination-null/Dockerfile | 11 ++ .../connectors/destination-null/README.md | 68 ++++++++++ .../connectors/destination-null/build.gradle | 19 +++ .../destination_null/NullConsumer.java | 76 +++++++++++ .../destination_null/NullDestination.java | 43 ++++++ .../destination_null/logger/BaseLogger.java | 45 +++++++ .../destination_null/logger/DevNull.java | 17 +++ .../logger/EveryNthLogger.java | 33 +++++ .../destination_null/logger/FirstNLogger.java | 26 ++++ .../logger/NullDestinationLogger.java | 16 +++ .../logger/NullDestinationLoggerFactory.java | 45 +++++++ .../logger/RandomSamplingLogger.java | 34 +++++ .../src/main/resources/spec.json | 123 ++++++++++++++++++ .../NullDestinationAcceptanceTest.java | 65 +++++++++ docs/integrations/destinations/null.md | 52 ++++++++ 16 files changed, 676 insertions(+) create mode 100644 airbyte-integrations/connectors/destination-null/.dockerignore create mode 100644 airbyte-integrations/connectors/destination-null/Dockerfile create mode 100644 airbyte-integrations/connectors/destination-null/README.md create mode 100644 airbyte-integrations/connectors/destination-null/build.gradle create mode 100644 airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/NullConsumer.java create mode 100644 airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/NullDestination.java create mode 100644 airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/BaseLogger.java create mode 100644 airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/DevNull.java create mode 100644 airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/EveryNthLogger.java create mode 100644 airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/FirstNLogger.java create mode 100644 airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/NullDestinationLogger.java create mode 100644 airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/NullDestinationLoggerFactory.java create mode 100644 airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/RandomSamplingLogger.java create mode 100644 airbyte-integrations/connectors/destination-null/src/main/resources/spec.json create mode 100644 airbyte-integrations/connectors/destination-null/src/test-integration/java/io/airbyte/integrations/destination/destination_null/NullDestinationAcceptanceTest.java create mode 100644 docs/integrations/destinations/null.md diff --git a/airbyte-integrations/connectors/destination-null/.dockerignore b/airbyte-integrations/connectors/destination-null/.dockerignore new file mode 100644 index 000000000000..65c7d0ad3e73 --- /dev/null +++ b/airbyte-integrations/connectors/destination-null/.dockerignore @@ -0,0 +1,3 @@ +* +!Dockerfile +!build diff --git a/airbyte-integrations/connectors/destination-null/Dockerfile b/airbyte-integrations/connectors/destination-null/Dockerfile new file mode 100644 index 000000000000..24f089085779 --- /dev/null +++ b/airbyte-integrations/connectors/destination-null/Dockerfile @@ -0,0 +1,11 @@ +FROM airbyte/integration-base-java:dev + +WORKDIR /airbyte +ENV APPLICATION destination-null + +COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar + +RUN tar xf ${APPLICATION}.tar --strip-components=1 + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/destination-null diff --git a/airbyte-integrations/connectors/destination-null/README.md b/airbyte-integrations/connectors/destination-null/README.md new file mode 100644 index 000000000000..3079fc9aa19e --- /dev/null +++ b/airbyte-integrations/connectors/destination-null/README.md @@ -0,0 +1,68 @@ +# Destination Null + +This is the repository for the Null destination connector in Java. +For information about how to use this connector within Airbyte, see [the User Documentation](https://docs.airbyte.io/integrations/destinations/null). + +## Local development + +#### Building via Gradle +From the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:destination-null:build +``` + +#### Create credentials +**If you are a community contributor**, generate the necessary credentials and place them in `secrets/config.json` conforming to the spec file in `src/main/resources/spec.json`. +Note that the `secrets` directory is git-ignored by default, so there is no danger of accidentally checking in sensitive information. + +**If you are an Airbyte core member**, follow the [instructions](https://docs.airbyte.io/connector-development#using-credentials-in-ci) to set up the credentials. + +### Locally running the connector docker image + +#### Build +Build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:destination-null:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/destination-null:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-null:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-null:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-null:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +## Testing +We use `JUnit` for Java tests. + +### Unit and Integration Tests +Place unit tests under `src/test/io/airbyte/integrations/destinations/null`. + +#### Acceptance Tests +Airbyte has a standard test suite that all destination connectors must pass. Implement the `TODO`s in +`src/test-integration/java/io/airbyte/integrations/destinations/nullDestinationAcceptanceTest.java`. + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:destination-null:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:destination-null:integrationTest +``` + +## Dependency Management + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/destination-null/build.gradle b/airbyte-integrations/connectors/destination-null/build.gradle new file mode 100644 index 000000000000..2aaa93d11eb3 --- /dev/null +++ b/airbyte-integrations/connectors/destination-null/build.gradle @@ -0,0 +1,19 @@ +plugins { + id 'application' + id 'airbyte-docker' + id 'airbyte-integration-test-java' +} + +application { + mainClass = 'io.airbyte.integrations.destination.destination_null.NullDestination' +} + +dependencies { + implementation project(':airbyte-config:models') + implementation project(':airbyte-protocol:models') + implementation project(':airbyte-integrations:bases:base-java') + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + + integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') + integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-null') +} diff --git a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/NullConsumer.java b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/NullConsumer.java new file mode 100644 index 000000000000..ed45c356b6e2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/NullConsumer.java @@ -0,0 +1,76 @@ +package io.airbyte.integrations.destination.destination_null; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.integrations.destination.destination_null.logger.NullDestinationLogger; +import io.airbyte.integrations.destination.destination_null.logger.NullDestinationLoggerFactory; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.AirbyteMessage.Type; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import io.airbyte.protocol.models.AirbyteStream; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +public class NullConsumer extends FailureTrackingAirbyteMessageConsumer { + + private final NullDestinationLoggerFactory loggerFactory; + private final ConfiguredAirbyteCatalog configuredCatalog; + private final Consumer outputRecordCollector; + private final Map loggers; + + private AirbyteMessage lastStateMessage = null; + + public NullConsumer(NullDestinationLoggerFactory loggerFactory, + ConfiguredAirbyteCatalog configuredCatalog, + Consumer outputRecordCollector) { + this.loggerFactory = loggerFactory; + this.configuredCatalog = configuredCatalog; + this.outputRecordCollector = outputRecordCollector; + this.loggers = new HashMap<>(); + } + + @Override + protected void startTracked() { + for (ConfiguredAirbyteStream configuredStream : configuredCatalog.getStreams()) { + final AirbyteStream stream = configuredStream.getStream(); + final AirbyteStreamNameNamespacePair streamNamePair = AirbyteStreamNameNamespacePair.fromAirbyteSteam(stream); + final NullDestinationLogger logger = loggerFactory.create(streamNamePair); + loggers.put(streamNamePair, logger); + } + } + + @Override + protected void acceptTracked(AirbyteMessage airbyteMessage) { + if (airbyteMessage.getType() == Type.STATE) { + this.lastStateMessage = airbyteMessage; + return; + } else if (airbyteMessage.getType() != Type.RECORD) { + return; + } + + AirbyteRecordMessage recordMessage = airbyteMessage.getRecord(); + AirbyteStreamNameNamespacePair pair = AirbyteStreamNameNamespacePair + .fromRecordMessage(recordMessage); + + if (!loggers.containsKey(pair)) { + throw new IllegalArgumentException( + String.format( + "Message contained record from a stream that was not in the catalog. \ncatalog: %s , \nmessage: %s", + Jsons.serialize(configuredCatalog), Jsons.serialize(recordMessage))); + } + + loggers.get(pair).log(recordMessage); + } + + @Override + protected void close(boolean hasFailed) { + if (!hasFailed) { + outputRecordCollector.accept(lastStateMessage); + } + } + +} diff --git a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/NullDestination.java b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/NullDestination.java new file mode 100644 index 000000000000..13fca00407e4 --- /dev/null +++ b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/NullDestination.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.destination_null; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.integrations.BaseConnector; +import io.airbyte.integrations.base.AirbyteMessageConsumer; +import io.airbyte.integrations.base.Destination; +import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.integrations.destination.destination_null.logger.NullDestinationLogger; +import io.airbyte.integrations.destination.destination_null.logger.NullDestinationLoggerFactory; +import io.airbyte.protocol.models.AirbyteConnectionStatus; +import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import java.util.function.Consumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class NullDestination extends BaseConnector implements Destination { + + public static void main(String[] args) throws Exception { + new IntegrationRunner(new NullDestination()).run(args); + } + + /** + * The null destination is always available! + */ + @Override + public AirbyteConnectionStatus check(JsonNode config) { + return new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED); + } + + @Override + public AirbyteMessageConsumer getConsumer(JsonNode config, + ConfiguredAirbyteCatalog configuredCatalog, + Consumer outputRecordCollector) { + return new NullConsumer(new NullDestinationLoggerFactory(config), configuredCatalog, outputRecordCollector); + } + +} diff --git a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/BaseLogger.java b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/BaseLogger.java new file mode 100644 index 000000000000..df65e9f2be40 --- /dev/null +++ b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/BaseLogger.java @@ -0,0 +1,45 @@ +package io.airbyte.integrations.destination.destination_null.logger; + +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class BaseLogger implements NullDestinationLogger { + + protected final AirbyteStreamNameNamespacePair streamNamePair; + protected final int maxEntryCount; + protected int loggedEntryCount = 0; + + public BaseLogger(AirbyteStreamNameNamespacePair streamNamePair, int maxEntryCount) { + this.streamNamePair = streamNamePair; + this.maxEntryCount = maxEntryCount; + } + + protected String entryMessage(AirbyteRecordMessage recordMessage) { + return String.format("[%s] %s #%04d: %s", + emissionTimestamp(recordMessage.getEmittedAt()), + streamName(streamNamePair), + loggedEntryCount, + recordMessage.getData()); + } + + protected static String streamName(AirbyteStreamNameNamespacePair pair) { + if (pair.getNamespace() == null) { + return pair.getName(); + } else { + return String.format("%s.%s", pair.getNamespace(), pair.getName()); + } + } + + protected static String emissionTimestamp(long emittedAt) { + return OffsetDateTime + .ofInstant(Instant.ofEpochMilli(emittedAt), ZoneId.systemDefault()) + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); + } + +} diff --git a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/DevNull.java b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/DevNull.java new file mode 100644 index 000000000000..a627cfa44f59 --- /dev/null +++ b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/DevNull.java @@ -0,0 +1,17 @@ +package io.airbyte.integrations.destination.destination_null.logger; + +import io.airbyte.protocol.models.AirbyteRecordMessage; + +public class DevNull implements NullDestinationLogger { + + public static final DevNull SINGLETON = new DevNull(); + + private DevNull() { + } + + @Override + public void log(AirbyteRecordMessage recordMessage) { + // nothing happens in /dev/null + } + +} diff --git a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/EveryNthLogger.java b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/EveryNthLogger.java new file mode 100644 index 000000000000..5d645e888dac --- /dev/null +++ b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/EveryNthLogger.java @@ -0,0 +1,33 @@ +package io.airbyte.integrations.destination.destination_null.logger; + +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class EveryNthLogger extends BaseLogger implements NullDestinationLogger { + + private static final Logger LOGGER = LoggerFactory.getLogger(EveryNthLogger.class); + + private final int nthEntryToLog; + private int currentEntry = 0; + + public EveryNthLogger(AirbyteStreamNameNamespacePair streamNamePair, int nthEntryToLog, int maxEntryCount) { + super(streamNamePair, maxEntryCount); + this.nthEntryToLog = nthEntryToLog; + } + + @Override + public void log(AirbyteRecordMessage recordMessage) { + if (loggedEntryCount >= maxEntryCount) { + return; + } + + currentEntry += 1; + if (currentEntry % nthEntryToLog == 0) { + loggedEntryCount += 1; + LOGGER.info(entryMessage(recordMessage)); + } + } + +} diff --git a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/FirstNLogger.java b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/FirstNLogger.java new file mode 100644 index 000000000000..a394c657b479 --- /dev/null +++ b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/FirstNLogger.java @@ -0,0 +1,26 @@ +package io.airbyte.integrations.destination.destination_null.logger; + +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FirstNLogger extends BaseLogger implements NullDestinationLogger { + + private static final Logger LOGGER = LoggerFactory.getLogger(FirstNLogger.class); + + public FirstNLogger(AirbyteStreamNameNamespacePair streamNamePair, int maxEntryCount) { + super(streamNamePair, maxEntryCount); + } + + @Override + public void log(AirbyteRecordMessage recordMessage) { + if (loggedEntryCount >= maxEntryCount) { + return; + } + + loggedEntryCount += 1; + LOGGER.info(entryMessage(recordMessage)); + } + +} diff --git a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/NullDestinationLogger.java b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/NullDestinationLogger.java new file mode 100644 index 000000000000..627ab41b46a0 --- /dev/null +++ b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/NullDestinationLogger.java @@ -0,0 +1,16 @@ +package io.airbyte.integrations.destination.destination_null.logger; + +import io.airbyte.protocol.models.AirbyteRecordMessage; + +public interface NullDestinationLogger { + + enum LoggingType { + NoLogging, + FirstN, + EveryNth, + RandomSampling + } + + void log(AirbyteRecordMessage recordMessage); + +} diff --git a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/NullDestinationLoggerFactory.java b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/NullDestinationLoggerFactory.java new file mode 100644 index 000000000000..0818264b32f8 --- /dev/null +++ b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/NullDestinationLoggerFactory.java @@ -0,0 +1,45 @@ +package io.airbyte.integrations.destination.destination_null.logger; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.integrations.destination.destination_null.logger.NullDestinationLogger.LoggingType; + +public class NullDestinationLoggerFactory { + + private final JsonNode config; + + public NullDestinationLoggerFactory(JsonNode config) { + this.config = config; + } + + public NullDestinationLogger create(AirbyteStreamNameNamespacePair streamNamePair) { + if (!config.has("logging")) { + throw new IllegalArgumentException("Property logging is required, but not found"); + } + + final JsonNode logConfig = config.get("logging"); + final LoggingType loggingType = LoggingType.valueOf(logConfig.get("logging_type").asText()); + switch (loggingType) { + case NoLogging -> { + return DevNull.SINGLETON; + } + case FirstN -> { + final int maxEntryCount = logConfig.get("max_entry_count").asInt(); + return new FirstNLogger(streamNamePair, maxEntryCount); + } + case EveryNth -> { + final int nthEntryToLog = logConfig.get("nth_entry_to_log").asInt(); + final int maxEntryCount = logConfig.get("max_entry_count").asInt(); + return new EveryNthLogger(streamNamePair, nthEntryToLog, maxEntryCount); + } + case RandomSampling -> { + final double samplingRatio = logConfig.get("sampling_ratio").asDouble(); + final long seed = logConfig.has("seed") ? logConfig.get("seed").asLong() : System.currentTimeMillis(); + final int maxEntryCount = logConfig.get("max_entry_count").asInt(); + return new RandomSamplingLogger(streamNamePair, samplingRatio, seed, maxEntryCount); + } + default -> throw new IllegalArgumentException("Unexpected logging type: " + loggingType); + } + } + +} diff --git a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/RandomSamplingLogger.java b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/RandomSamplingLogger.java new file mode 100644 index 000000000000..aecb98d4f03d --- /dev/null +++ b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/RandomSamplingLogger.java @@ -0,0 +1,34 @@ +package io.airbyte.integrations.destination.destination_null.logger; + +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import java.util.Random; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RandomSamplingLogger extends BaseLogger implements NullDestinationLogger { + + private static final Logger LOGGER = LoggerFactory.getLogger(RandomSamplingLogger.class); + + private final double samplingRatio; + private final Random random; + + public RandomSamplingLogger(AirbyteStreamNameNamespacePair streamNamePair, double samplingRatio, long seed, int maxEntryCount) { + super(streamNamePair, maxEntryCount); + this.samplingRatio = samplingRatio; + this.random = new Random(seed); + } + + @Override + public void log(AirbyteRecordMessage recordMessage) { + if (loggedEntryCount >= maxEntryCount) { + return; + } + + if (random.nextDouble() < samplingRatio) { + loggedEntryCount += 1; + LOGGER.info(entryMessage(recordMessage)); + } + } + +} diff --git a/airbyte-integrations/connectors/destination-null/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-null/src/main/resources/spec.json new file mode 100644 index 000000000000..55c96bba3a3e --- /dev/null +++ b/airbyte-integrations/connectors/destination-null/src/main/resources/spec.json @@ -0,0 +1,123 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/destinations/null", + "supportsIncremental": true, + "supported_destination_sync_modes": ["overwrite", "append"], + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Null Destination Spec", + "type": "object", + "required": ["logging"], + "additionalProperties": false, + "properties": { + "logging": { + "title": "Log Output", + "type": "object", + "description": "Whether the data should be logged.", + "oneOf": [ + { + "title": "No Logging", + "description": "Log nothing. A pure /dev/null.", + "type": "object", + "required": ["logging_type"], + "properties": { + "logging_type": { + "type": "string", + "enum": ["NoLogging"], + "default": "NoLogging" + } + } + }, + { + "title": "First N Entries", + "description": "Log first N entries per stream.", + "type": "object", + "required": ["logging_type", "max_entry_count"], + "properties": { + "logging_type": { + "type": "string", + "enum": ["FirstN"], + "default": "FirstN" + }, + "max_entry_count": { + "title": "N", + "description": "Number of entries to log. This destination is for testing only. So it won't make sense to log infinitely. The maximum is 1,000 entries.", + "type": "number", + "default": 100, + "examples": [100], + "minimum": 1, + "maximum": 1000 + } + } + }, + { + "title": "Every N-th Entry", + "description": "For each stream, log every N-th entry with a maximum cap.", + "type": "object", + "required": ["logging_type", "nth_entry_to_log", "max_entry_count"], + "properties": { + "logging_type": { + "type": "string", + "enum": ["EveryNth"], + "default": "EveryNth" + }, + "nth_entry_to_log": { + "title": "N", + "description": "The N-th entry to log for each stream. N starts from 1. For example, when N = 1, every entry is logged; when N = 2, every other entry is logged; when N = 3, one out of three entries is logged.", + "type": "number", + "example": [3], + "minimum": 1, + "maximum": 1000 + }, + "max_entry_count": { + "title": "Max Log Entries", + "description": "Max number of entries to log. This destination is for testing only. So it won't make sense to log infinitely. The maximum is 1,000 entries.", + "type": "number", + "default": 100, + "examples": [100], + "minimum": 1, + "maximum": 1000 + } + } + }, + { + "title": "Random Sampling", + "description": "For each stream, randomly log a percentage of the entries with a maximum cap.", + "type": "object", + "required": ["logging_type", "sampling_ratio", "max_entry_count"], + "properties": { + "logging_type": { + "type": "string", + "enum": ["RandomSampling"], + "default": "RandomSampling" + }, + "sampling_ratio": { + "title": "Sampling Ratio", + "description": "A positive floating number smaller than 1.", + "type": "number", + "default": 0.001, + "examples": [0.001], + "minimum": 0, + "maximum": 1 + }, + "seed": { + "title": "Random Number Generator Seed", + "description": "When the seed is unspecified, the current time millis will be used as the seed.", + "type": "number", + "examples": [1900] + }, + "max_entry_count": { + "title": "Max Log Entries", + "description": "Max number of entries to log. This destination is for testing only. So it won't make sense to log infinitely. The maximum is 1,000 entries.", + "type": "number", + "default": 100, + "examples": [100], + "minimum": 1, + "maximum": 1000 + } + } + } + ] + } + } + } +} diff --git a/airbyte-integrations/connectors/destination-null/src/test-integration/java/io/airbyte/integrations/destination/destination_null/NullDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-null/src/test-integration/java/io/airbyte/integrations/destination/destination_null/NullDestinationAcceptanceTest.java new file mode 100644 index 000000000000..6a1263b42573 --- /dev/null +++ b/airbyte-integrations/connectors/destination-null/src/test-integration/java/io/airbyte/integrations/destination/destination_null/NullDestinationAcceptanceTest.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.destination_null; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import java.util.Collections; +import java.util.List; + +public class NullDestinationAcceptanceTest extends DestinationAcceptanceTest { + + @Override + protected String getImageName() { + return "airbyte/destination-null:dev"; + } + + @Override + protected JsonNode getConfig() { + final JsonNode loggingConfig = Jsons.jsonNode(ImmutableMap.builder() + .put("logging_type", "EveryNth") + .put("nth_entry_to_log", 1) + .put("max_entry_count", 3) + .build()); + return Jsons.jsonNode(Collections.singletonMap("logging", loggingConfig)); + } + + @Override + protected JsonNode getFailCheckConfig() { + final JsonNode loggingConfig = Jsons.jsonNode(ImmutableMap.builder() + .put("logging_type", "LastN") + .put("max_entry_count", 3000) + .build()); + return Jsons.jsonNode(Collections.singletonMap("logging", loggingConfig)); + } + + @Override + protected List retrieveRecords(TestDestinationEnv testEnv, + String streamName, + String namespace, + JsonNode streamSchema) { + return Collections.emptyList(); + } + + @Override + protected void setup(TestDestinationEnv testEnv) { + // do nothing + } + + @Override + protected void tearDown(TestDestinationEnv testEnv) { + // do nothing + } + + @Override + protected void assertSameMessages(List expected, List actual, boolean pruneAirbyteInternalFields) { + // do nothing + } + +} diff --git a/docs/integrations/destinations/null.md b/docs/integrations/destinations/null.md new file mode 100644 index 000000000000..966555a7c4bd --- /dev/null +++ b/docs/integrations/destinations/null.md @@ -0,0 +1,52 @@ +# Null + +TODO: update this doc + +## Sync overview + +### Output schema + +Is the output schema fixed (e.g: for an API like Stripe)? If so, point to the connector's schema (e.g: link to Stripe’s documentation) or describe the schema here directly (e.g: include a diagram or paragraphs describing the schema). + +Describe how the connector's schema is mapped to Airbyte concepts. An example description might be: "MagicDB tables become Airbyte Streams and MagicDB columns become Airbyte Fields. In addition, an extracted\_at column is appended to each row being read." + +### Data type mapping + +This section should contain a table mapping each of the connector's data types to Airbyte types. At the moment, Airbyte uses the same types used by [JSONSchema](https://json-schema.org/understanding-json-schema/reference/index.html). `string`, `date-time`, `object`, `array`, `boolean`, `integer`, and `number` are the most commonly used data types. + +| Integration Type | Airbyte Type | Notes | +| :--- | :--- | :--- | + + +### Features + +This section should contain a table with the following format: + +| Feature | Supported?(Yes/No) | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | | | +| Incremental Sync | | | +| Replicate Incremental Deletes | | | +| For databases, WAL/Logical replication | | | +| SSL connection | | | +| SSH Tunnel Support | | | +| (Any other source-specific features) | | | + +### Performance considerations + +Could this connector hurt the user's database/API/etc... or put too much strain on it in certain circumstances? For example, if there are a lot of tables or rows in a table? What is the breaking point (e.g: 100mm> records)? What can the user do to prevent this? (e.g: use a read-only replica, or schedule frequent syncs, etc..) + +## Getting started + +### Requirements + +* What versions of this connector does this implementation support? (e.g: `postgres v3.14 and above`) +* What configurations, if any, are required on the connector? (e.g: `buffer_size > 1024`) +* Network accessibility requirements +* Credentials/authentication requirements? (e.g: A DB user with read permissions on certain tables) + +### Setup guide + +For each of the above high-level requirements as appropriate, add or point to a follow-along guide. See existing source or destination guides for an example. + +For each major cloud provider we support, also add a follow-along guide for setting up Airbyte to connect to that destination. See the Postgres destination guide for an example of what this should look like. From 3d4fd89b2299514aee15742cde6c8f6862a5a97a Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Thu, 16 Dec 2021 16:55:44 -0800 Subject: [PATCH 02/17] Update existing testing destinations --- .../destination/e2e_test/FailAfterNDestination.java | 7 +++---- .../destination/e2e_test/LoggingDestination.java | 2 +- .../destination/e2e_test/SilentDestination.java | 12 ------------ .../destination/e2e_test/ThrottledDestination.java | 5 ++--- 4 files changed, 6 insertions(+), 20 deletions(-) diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/FailAfterNDestination.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/FailAfterNDestination.java index 4a4a577b5eba..59a73b68e5b8 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/FailAfterNDestination.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/FailAfterNDestination.java @@ -43,23 +43,22 @@ public FailAfterNConsumer(final long numMessagesAfterWhichToFail, final Consumer this.numMessagesAfterWhichToFail = numMessagesAfterWhichToFail; this.outputRecordCollector = outputRecordCollector; this.numMessagesSoFar = 0; + LOGGER.info("Will fail after {} messages", numMessagesAfterWhichToFail); } @Override public void start() {} @Override - public void accept(final AirbyteMessage message) throws Exception { - LOGGER.info("received record: {}", message); + public void accept(final AirbyteMessage message) { numMessagesSoFar += 1; - LOGGER.info("received {} messages so far", numMessagesSoFar); if (numMessagesSoFar > numMessagesAfterWhichToFail) { throw new IllegalStateException("Forcing a fail after processing " + numMessagesAfterWhichToFail + " messages."); } if (message.getType() == Type.STATE) { - LOGGER.info("emitting state: {}", message); + LOGGER.info("Emitting state: {}", message); outputRecordCollector.accept(message); } } diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/LoggingDestination.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/LoggingDestination.java index 0d456aa6b5c0..51e0e68f4290 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/LoggingDestination.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/LoggingDestination.java @@ -52,7 +52,7 @@ public void accept(final AirbyteMessage message) { LOGGER.info("record: {}", message); if (message.getType() == Type.STATE) { - LOGGER.info("emitting state: {}", message); + LOGGER.info("Emitting state: {}", message); outputRecordCollector.accept(message); } } diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/SilentDestination.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/SilentDestination.java index f53108d98f64..aec6c3b1e680 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/SilentDestination.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/SilentDestination.java @@ -13,19 +13,13 @@ import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * This destination silently receives records. */ public class SilentDestination extends BaseConnector implements Destination { - private static final Logger LOGGER = LoggerFactory.getLogger(LoggingDestination.class); - private static final AtomicInteger counter = new AtomicInteger(); - @Override public AirbyteConnectionStatus check(final JsonNode config) { return new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED); @@ -52,14 +46,8 @@ public void start() {} @Override public void accept(final AirbyteMessage message) { if (message.getType() == Type.STATE) { - LOGGER.info("emitting state: {}", message); outputRecordCollector.accept(message); } - - final var curr = counter.incrementAndGet(); - if (curr % 1000 == 0) { - LOGGER.info("Record count: {}", curr); - } } @Override diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/ThrottledDestination.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/ThrottledDestination.java index 36a65cc77c57..b3be0d18c6e9 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/ThrottledDestination.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/ThrottledDestination.java @@ -47,6 +47,7 @@ public static class ThrottledConsumer implements AirbyteMessageConsumer { public ThrottledConsumer(final long millisPerRecord, final Consumer outputRecordCollector) { this.millisPerRecord = millisPerRecord; this.outputRecordCollector = outputRecordCollector; + LOGGER.info("Will sleep {} millis before processing every record", millisPerRecord); } @Override @@ -54,12 +55,10 @@ public void start() {} @Override public void accept(final AirbyteMessage message) throws Exception { - LOGGER.info("received record: {}", message); sleep(millisPerRecord); - LOGGER.info("completed sleep"); if (message.getType() == Type.STATE) { - LOGGER.info("emitting state: {}", message); + LOGGER.info("Emitting state: {}", message); outputRecordCollector.accept(message); } } From 7c7892866e017463203fdca7a42895dd5a780d3d Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Thu, 16 Dec 2021 17:49:50 -0800 Subject: [PATCH 03/17] Merge in logging consumer --- .../e2e_test/LoggingDestination.java | 31 +----- .../e2e_test/logging/BaseLogger.java | 43 ++++++++ .../e2e_test/logging/EveryNthLogger.java | 33 ++++++ .../e2e_test/logging/FirstNLogger.java | 26 +++++ .../e2e_test/logging/LoggingConsumer.java | 74 +++++++++++++ .../logging/RandomSamplingLogger.java | 34 ++++++ .../e2e_test/logging/TestingLogger.java | 15 +++ .../logging/TestingLoggerFactory.java | 42 ++++++++ .../src/main/resources/spec.json | 100 +++++++++++++++++- ...tinationLoggingEveryNthAcceptanceTest.java | 66 ++++++++++++ 10 files changed, 434 insertions(+), 30 deletions(-) create mode 100644 airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/BaseLogger.java create mode 100644 airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/EveryNthLogger.java create mode 100644 airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/FirstNLogger.java create mode 100644 airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.java create mode 100644 airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/RandomSamplingLogger.java create mode 100644 airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/TestingLogger.java create mode 100644 airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/TestingLoggerFactory.java create mode 100644 airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingDestinationLoggingEveryNthAcceptanceTest.java diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/LoggingDestination.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/LoggingDestination.java index 51e0e68f4290..171f73ed8dd1 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/LoggingDestination.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/LoggingDestination.java @@ -8,10 +8,11 @@ import io.airbyte.integrations.BaseConnector; import io.airbyte.integrations.base.AirbyteMessageConsumer; import io.airbyte.integrations.base.Destination; +import io.airbyte.integrations.destination.e2e_test.logging.LoggingConsumer; +import io.airbyte.integrations.destination.e2e_test.logging.TestingLoggerFactory; import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.AirbyteMessage; -import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import java.util.function.Consumer; import org.slf4j.Logger; @@ -33,33 +34,7 @@ public AirbyteConnectionStatus check(final JsonNode config) { public AirbyteMessageConsumer getConsumer(final JsonNode config, final ConfiguredAirbyteCatalog catalog, final Consumer outputRecordCollector) { - return new RecordConsumer(outputRecordCollector); - } - - public static class RecordConsumer implements AirbyteMessageConsumer { - - private final Consumer outputRecordCollector; - - public RecordConsumer(final Consumer outputRecordCollector) { - this.outputRecordCollector = outputRecordCollector; - } - - @Override - public void start() {} - - @Override - public void accept(final AirbyteMessage message) { - LOGGER.info("record: {}", message); - - if (message.getType() == Type.STATE) { - LOGGER.info("Emitting state: {}", message); - outputRecordCollector.accept(message); - } - } - - @Override - public void close() {} - + return new LoggingConsumer(new TestingLoggerFactory(config), catalog, outputRecordCollector); } } diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/BaseLogger.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/BaseLogger.java new file mode 100644 index 000000000000..596369366463 --- /dev/null +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/BaseLogger.java @@ -0,0 +1,43 @@ +package io.airbyte.integrations.destination.e2e_test.logging; + +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; + +public abstract class BaseLogger implements TestingLogger { + + protected final AirbyteStreamNameNamespacePair streamNamePair; + protected final int maxEntryCount; + protected int loggedEntryCount = 0; + + public BaseLogger(final AirbyteStreamNameNamespacePair streamNamePair, final int maxEntryCount) { + this.streamNamePair = streamNamePair; + this.maxEntryCount = maxEntryCount; + } + + protected String entryMessage(final AirbyteRecordMessage recordMessage) { + return String.format("[%s] %s #%04d: %s", + emissionTimestamp(recordMessage.getEmittedAt()), + streamName(streamNamePair), + loggedEntryCount, + recordMessage.getData()); + } + + protected static String streamName(final AirbyteStreamNameNamespacePair pair) { + if (pair.getNamespace() == null) { + return pair.getName(); + } else { + return String.format("%s.%s", pair.getNamespace(), pair.getName()); + } + } + + protected static String emissionTimestamp(final long emittedAt) { + return OffsetDateTime + .ofInstant(Instant.ofEpochMilli(emittedAt), ZoneId.systemDefault()) + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); + } + +} diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/EveryNthLogger.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/EveryNthLogger.java new file mode 100644 index 000000000000..50fd4089551f --- /dev/null +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/EveryNthLogger.java @@ -0,0 +1,33 @@ +package io.airbyte.integrations.destination.e2e_test.logging; + +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class EveryNthLogger extends BaseLogger implements TestingLogger { + + private static final Logger LOGGER = LoggerFactory.getLogger(EveryNthLogger.class); + + private final int nthEntryToLog; + private int currentEntry = 0; + + public EveryNthLogger(final AirbyteStreamNameNamespacePair streamNamePair, final int nthEntryToLog, final int maxEntryCount) { + super(streamNamePair, maxEntryCount); + this.nthEntryToLog = nthEntryToLog; + } + + @Override + public void log(final AirbyteRecordMessage recordMessage) { + if (loggedEntryCount >= maxEntryCount) { + return; + } + + currentEntry += 1; + if (currentEntry % nthEntryToLog == 0) { + loggedEntryCount += 1; + LOGGER.info(entryMessage(recordMessage)); + } + } + +} diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/FirstNLogger.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/FirstNLogger.java new file mode 100644 index 000000000000..10d873f90349 --- /dev/null +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/FirstNLogger.java @@ -0,0 +1,26 @@ +package io.airbyte.integrations.destination.e2e_test.logging; + +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FirstNLogger extends BaseLogger implements TestingLogger { + + private static final Logger LOGGER = LoggerFactory.getLogger(FirstNLogger.class); + + public FirstNLogger(final AirbyteStreamNameNamespacePair streamNamePair, final int maxEntryCount) { + super(streamNamePair, maxEntryCount); + } + + @Override + public void log(final AirbyteRecordMessage recordMessage) { + if (loggedEntryCount >= maxEntryCount) { + return; + } + + loggedEntryCount += 1; + LOGGER.info(entryMessage(recordMessage)); + } + +} diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.java new file mode 100644 index 000000000000..4fb3870ae548 --- /dev/null +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.java @@ -0,0 +1,74 @@ +package io.airbyte.integrations.destination.e2e_test.logging; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.AirbyteMessage.Type; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import io.airbyte.protocol.models.AirbyteStream; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +public class LoggingConsumer extends FailureTrackingAirbyteMessageConsumer { + + private final TestingLoggerFactory loggerFactory; + private final ConfiguredAirbyteCatalog configuredCatalog; + private final Consumer outputRecordCollector; + private final Map loggers; + + private AirbyteMessage lastStateMessage = null; + + public LoggingConsumer(final TestingLoggerFactory loggerFactory, + final ConfiguredAirbyteCatalog configuredCatalog, + final Consumer outputRecordCollector) { + this.loggerFactory = loggerFactory; + this.configuredCatalog = configuredCatalog; + this.outputRecordCollector = outputRecordCollector; + this.loggers = new HashMap<>(); + } + + @Override + protected void startTracked() { + for (final ConfiguredAirbyteStream configuredStream : configuredCatalog.getStreams()) { + final AirbyteStream stream = configuredStream.getStream(); + final AirbyteStreamNameNamespacePair streamNamePair = AirbyteStreamNameNamespacePair.fromAirbyteSteam(stream); + final TestingLogger logger = loggerFactory.create(streamNamePair); + loggers.put(streamNamePair, logger); + } + } + + @Override + protected void acceptTracked(final AirbyteMessage airbyteMessage) { + if (airbyteMessage.getType() == Type.STATE) { + this.lastStateMessage = airbyteMessage; + return; + } else if (airbyteMessage.getType() != Type.RECORD) { + return; + } + + final AirbyteRecordMessage recordMessage = airbyteMessage.getRecord(); + final AirbyteStreamNameNamespacePair pair = AirbyteStreamNameNamespacePair + .fromRecordMessage(recordMessage); + + if (!loggers.containsKey(pair)) { + throw new IllegalArgumentException( + String.format( + "Message contained record from a stream that was not in the catalog. \ncatalog: %s , \nmessage: %s", + Jsons.serialize(configuredCatalog), Jsons.serialize(recordMessage))); + } + + loggers.get(pair).log(recordMessage); + } + + @Override + protected void close(final boolean hasFailed) { + if (!hasFailed) { + outputRecordCollector.accept(lastStateMessage); + } + } + +} diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/RandomSamplingLogger.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/RandomSamplingLogger.java new file mode 100644 index 000000000000..b447d012f0c6 --- /dev/null +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/RandomSamplingLogger.java @@ -0,0 +1,34 @@ +package io.airbyte.integrations.destination.e2e_test.logging; + +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import java.util.Random; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RandomSamplingLogger extends BaseLogger implements TestingLogger { + + private static final Logger LOGGER = LoggerFactory.getLogger(RandomSamplingLogger.class); + + private final double samplingRatio; + private final Random random; + + public RandomSamplingLogger(final AirbyteStreamNameNamespacePair streamNamePair, final double samplingRatio, final long seed, final int maxEntryCount) { + super(streamNamePair, maxEntryCount); + this.samplingRatio = samplingRatio; + this.random = new Random(seed); + } + + @Override + public void log(final AirbyteRecordMessage recordMessage) { + if (loggedEntryCount >= maxEntryCount) { + return; + } + + if (random.nextDouble() < samplingRatio) { + loggedEntryCount += 1; + LOGGER.info(entryMessage(recordMessage)); + } + } + +} diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/TestingLogger.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/TestingLogger.java new file mode 100644 index 000000000000..edc91f1b8a74 --- /dev/null +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/TestingLogger.java @@ -0,0 +1,15 @@ +package io.airbyte.integrations.destination.e2e_test.logging; + +import io.airbyte.protocol.models.AirbyteRecordMessage; + +public interface TestingLogger { + + enum LoggingType { + FirstN, + EveryNth, + RandomSampling + } + + void log(AirbyteRecordMessage recordMessage); + +} diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/TestingLoggerFactory.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/TestingLoggerFactory.java new file mode 100644 index 000000000000..5f690b8c6514 --- /dev/null +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/TestingLoggerFactory.java @@ -0,0 +1,42 @@ +package io.airbyte.integrations.destination.e2e_test.logging; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.integrations.destination.e2e_test.logging.TestingLogger.LoggingType; + +public class TestingLoggerFactory { + + private final JsonNode config; + + public TestingLoggerFactory(final JsonNode config) { + this.config = config; + } + + public TestingLogger create(final AirbyteStreamNameNamespacePair streamNamePair) { + if (!config.has("logging_config")) { + throw new IllegalArgumentException("Property logging_config is required, but not found"); + } + + final JsonNode logConfig = config.get("logging_config"); + final LoggingType loggingType = LoggingType.valueOf(logConfig.get("logging_type").asText()); + switch (loggingType) { + case FirstN -> { + final int maxEntryCount = logConfig.get("max_entry_count").asInt(); + return new FirstNLogger(streamNamePair, maxEntryCount); + } + case EveryNth -> { + final int nthEntryToLog = logConfig.get("nth_entry_to_log").asInt(); + final int maxEntryCount = logConfig.get("max_entry_count").asInt(); + return new EveryNthLogger(streamNamePair, nthEntryToLog, maxEntryCount); + } + case RandomSampling -> { + final double samplingRatio = logConfig.get("sampling_ratio").asDouble(); + final long seed = logConfig.has("seed") ? logConfig.get("seed").asLong() : System.currentTimeMillis(); + final int maxEntryCount = logConfig.get("max_entry_count").asInt(); + return new RandomSamplingLogger(streamNamePair, samplingRatio, seed, maxEntryCount); + } + default -> throw new IllegalArgumentException("Unexpected logging type: " + loggingType); + } + } + +} diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json index 39df3bbafb29..2c0b9d83e3a7 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json @@ -11,13 +11,109 @@ "oneOf": [ { "title": "Logging", - "required": ["type"], + "required": ["type", "logging_config"], "additionalProperties": false, "properties": { "type": { "type": "string", "const": "LOGGING", "default": "LOGGING" + }, + "logging_config": { + "title": "Logging Configuration", + "type": "object", + "description": "Configurate how the messages are logged.", + "oneOf": [ + { + "title": "First N Entries", + "description": "Log first N entries per stream.", + "type": "object", + "required": ["logging_type", "max_entry_count"], + "properties": { + "logging_type": { + "type": "string", + "enum": ["FirstN"], + "default": "FirstN" + }, + "max_entry_count": { + "title": "N", + "description": "Number of entries to log. This destination is for testing only. So it won't make sense to log infinitely. The maximum is 1,000 entries.", + "type": "number", + "default": 100, + "examples": [100], + "minimum": 1, + "maximum": 1000 + } + } + }, + { + "title": "Every N-th Entry", + "description": "For each stream, log every N-th entry with a maximum cap.", + "type": "object", + "required": ["logging_type", "nth_entry_to_log", "max_entry_count"], + "properties": { + "logging_type": { + "type": "string", + "enum": ["EveryNth"], + "default": "EveryNth" + }, + "nth_entry_to_log": { + "title": "N", + "description": "The N-th entry to log for each stream. N starts from 1. For example, when N = 1, every entry is logged; when N = 2, every other entry is logged; when N = 3, one out of three entries is logged.", + "type": "number", + "example": [3], + "minimum": 1, + "maximum": 1000 + }, + "max_entry_count": { + "title": "Max Log Entries", + "description": "Max number of entries to log. This destination is for testing only. So it won't make sense to log infinitely. The maximum is 1,000 entries.", + "type": "number", + "default": 100, + "examples": [100], + "minimum": 1, + "maximum": 1000 + } + } + }, + { + "title": "Random Sampling", + "description": "For each stream, randomly log a percentage of the entries with a maximum cap.", + "type": "object", + "required": ["logging_type", "sampling_ratio", "max_entry_count"], + "properties": { + "logging_type": { + "type": "string", + "enum": ["RandomSampling"], + "default": "RandomSampling" + }, + "sampling_ratio": { + "title": "Sampling Ratio", + "description": "A positive floating number smaller than 1.", + "type": "number", + "default": 0.001, + "examples": [0.001], + "minimum": 0, + "maximum": 1 + }, + "seed": { + "title": "Random Number Generator Seed", + "description": "When the seed is unspecified, the current time millis will be used as the seed.", + "type": "number", + "examples": [1900] + }, + "max_entry_count": { + "title": "Max Log Entries", + "description": "Max number of entries to log. This destination is for testing only. So it won't make sense to log infinitely. The maximum is 1,000 entries.", + "type": "number", + "default": 100, + "examples": [100], + "minimum": 1, + "maximum": 1000 + } + } + } + ] } } }, @@ -44,7 +140,7 @@ "default": "THROTTLED" }, "millis_per_record": { - "description": "Time to pause in between records.", + "description": "Number of milli-second to pause in between records.", "type": "integer" } } diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingDestinationLoggingEveryNthAcceptanceTest.java b/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingDestinationLoggingEveryNthAcceptanceTest.java new file mode 100644 index 000000000000..3a3c9dba6c71 --- /dev/null +++ b/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingDestinationLoggingEveryNthAcceptanceTest.java @@ -0,0 +1,66 @@ +package io.airbyte.integrations.destination.e2e_test; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.destination.e2e_test.TestingDestinations.TestDestinationType; +import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class TestingDestinationLoggingEveryNthAcceptanceTest extends DestinationAcceptanceTest { + + @Override + protected String getImageName() { + return "airbyte/destination-e2e-test:dev"; + } + + @Override + protected JsonNode getConfig() { + final JsonNode loggingConfig = Jsons.jsonNode(ImmutableMap.builder() + .put("logging_type", "EveryNth") + .put("nth_entry_to_log", 1) + .put("max_entry_count", 3) + .build()); + return Jsons.jsonNode(Map.of("type", TestDestinationType.LOGGING.name(), "logging_config", loggingConfig)); + } + + @Override + protected JsonNode getFailCheckConfig() { + final JsonNode loggingConfig = Jsons.jsonNode(ImmutableMap.builder() + .put("logging_type", "LastN") + // max allowed entry count is 1000 + .put("max_entry_count", 3000) + .build()); + return Jsons.jsonNode(Map.of("type", TestDestinationType.LOGGING.name(), "logging_config", loggingConfig)); + } + + @Override + protected List retrieveRecords(final TestDestinationEnv testEnv, + final String streamName, + final String namespace, + final JsonNode streamSchema) { + return Collections.emptyList(); + } + + @Override + protected void setup(final TestDestinationEnv testEnv) { + // do nothing + } + + @Override + protected void tearDown(final TestDestinationEnv testEnv) { + // do nothing + } + + @Override + protected void assertSameMessages(final List expected, + final List actual, + final boolean pruneAirbyteInternalFields) { + // do nothing + } + +} From 169ba15de98e16cb2ba222d7a90fedec0a4085f6 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Thu, 16 Dec 2021 19:31:38 -0800 Subject: [PATCH 04/17] Remove old destination null --- .../connectors/destination-null/.dockerignore | 3 - .../connectors/destination-null/Dockerfile | 11 -- .../connectors/destination-null/README.md | 68 ---------- .../connectors/destination-null/build.gradle | 19 --- .../destination_null/NullConsumer.java | 76 ----------- .../destination_null/NullDestination.java | 43 ------ .../destination_null/logger/BaseLogger.java | 45 ------- .../destination_null/logger/DevNull.java | 17 --- .../logger/EveryNthLogger.java | 33 ----- .../destination_null/logger/FirstNLogger.java | 26 ---- .../logger/NullDestinationLogger.java | 16 --- .../logger/NullDestinationLoggerFactory.java | 45 ------- .../logger/RandomSamplingLogger.java | 34 ----- .../src/main/resources/spec.json | 123 ------------------ .../NullDestinationAcceptanceTest.java | 65 --------- 15 files changed, 624 deletions(-) delete mode 100644 airbyte-integrations/connectors/destination-null/.dockerignore delete mode 100644 airbyte-integrations/connectors/destination-null/Dockerfile delete mode 100644 airbyte-integrations/connectors/destination-null/README.md delete mode 100644 airbyte-integrations/connectors/destination-null/build.gradle delete mode 100644 airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/NullConsumer.java delete mode 100644 airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/NullDestination.java delete mode 100644 airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/BaseLogger.java delete mode 100644 airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/DevNull.java delete mode 100644 airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/EveryNthLogger.java delete mode 100644 airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/FirstNLogger.java delete mode 100644 airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/NullDestinationLogger.java delete mode 100644 airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/NullDestinationLoggerFactory.java delete mode 100644 airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/RandomSamplingLogger.java delete mode 100644 airbyte-integrations/connectors/destination-null/src/main/resources/spec.json delete mode 100644 airbyte-integrations/connectors/destination-null/src/test-integration/java/io/airbyte/integrations/destination/destination_null/NullDestinationAcceptanceTest.java diff --git a/airbyte-integrations/connectors/destination-null/.dockerignore b/airbyte-integrations/connectors/destination-null/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-null/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-null/Dockerfile b/airbyte-integrations/connectors/destination-null/Dockerfile deleted file mode 100644 index 24f089085779..000000000000 --- a/airbyte-integrations/connectors/destination-null/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte -ENV APPLICATION destination-null - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/destination-null diff --git a/airbyte-integrations/connectors/destination-null/README.md b/airbyte-integrations/connectors/destination-null/README.md deleted file mode 100644 index 3079fc9aa19e..000000000000 --- a/airbyte-integrations/connectors/destination-null/README.md +++ /dev/null @@ -1,68 +0,0 @@ -# Destination Null - -This is the repository for the Null destination connector in Java. -For information about how to use this connector within Airbyte, see [the User Documentation](https://docs.airbyte.io/integrations/destinations/null). - -## Local development - -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:destination-null:build -``` - -#### Create credentials -**If you are a community contributor**, generate the necessary credentials and place them in `secrets/config.json` conforming to the spec file in `src/main/resources/spec.json`. -Note that the `secrets` directory is git-ignored by default, so there is no danger of accidentally checking in sensitive information. - -**If you are an Airbyte core member**, follow the [instructions](https://docs.airbyte.io/connector-development#using-credentials-in-ci) to set up the credentials. - -### Locally running the connector docker image - -#### Build -Build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:destination-null:airbyteDocker -``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. - -#### Run -Then run any of the connector commands as follows: -``` -docker run --rm airbyte/destination-null:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-null:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-null:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-null:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json -``` - -## Testing -We use `JUnit` for Java tests. - -### Unit and Integration Tests -Place unit tests under `src/test/io/airbyte/integrations/destinations/null`. - -#### Acceptance Tests -Airbyte has a standard test suite that all destination connectors must pass. Implement the `TODO`s in -`src/test-integration/java/io/airbyte/integrations/destinations/nullDestinationAcceptanceTest.java`. - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:destination-null:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:destination-null:integrationTest -``` - -## Dependency Management - -### Publishing a new version of the connector -You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/destination-null/build.gradle b/airbyte-integrations/connectors/destination-null/build.gradle deleted file mode 100644 index 2aaa93d11eb3..000000000000 --- a/airbyte-integrations/connectors/destination-null/build.gradle +++ /dev/null @@ -1,19 +0,0 @@ -plugins { - id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' -} - -application { - mainClass = 'io.airbyte.integrations.destination.destination_null.NullDestination' -} - -dependencies { - implementation project(':airbyte-config:models') - implementation project(':airbyte-protocol:models') - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-null') -} diff --git a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/NullConsumer.java b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/NullConsumer.java deleted file mode 100644 index ed45c356b6e2..000000000000 --- a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/NullConsumer.java +++ /dev/null @@ -1,76 +0,0 @@ -package io.airbyte.integrations.destination.destination_null; - -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; -import io.airbyte.integrations.destination.destination_null.logger.NullDestinationLogger; -import io.airbyte.integrations.destination.destination_null.logger.NullDestinationLoggerFactory; -import io.airbyte.protocol.models.AirbyteMessage; -import io.airbyte.protocol.models.AirbyteMessage.Type; -import io.airbyte.protocol.models.AirbyteRecordMessage; -import io.airbyte.protocol.models.AirbyteStream; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.ConfiguredAirbyteStream; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Consumer; - -public class NullConsumer extends FailureTrackingAirbyteMessageConsumer { - - private final NullDestinationLoggerFactory loggerFactory; - private final ConfiguredAirbyteCatalog configuredCatalog; - private final Consumer outputRecordCollector; - private final Map loggers; - - private AirbyteMessage lastStateMessage = null; - - public NullConsumer(NullDestinationLoggerFactory loggerFactory, - ConfiguredAirbyteCatalog configuredCatalog, - Consumer outputRecordCollector) { - this.loggerFactory = loggerFactory; - this.configuredCatalog = configuredCatalog; - this.outputRecordCollector = outputRecordCollector; - this.loggers = new HashMap<>(); - } - - @Override - protected void startTracked() { - for (ConfiguredAirbyteStream configuredStream : configuredCatalog.getStreams()) { - final AirbyteStream stream = configuredStream.getStream(); - final AirbyteStreamNameNamespacePair streamNamePair = AirbyteStreamNameNamespacePair.fromAirbyteSteam(stream); - final NullDestinationLogger logger = loggerFactory.create(streamNamePair); - loggers.put(streamNamePair, logger); - } - } - - @Override - protected void acceptTracked(AirbyteMessage airbyteMessage) { - if (airbyteMessage.getType() == Type.STATE) { - this.lastStateMessage = airbyteMessage; - return; - } else if (airbyteMessage.getType() != Type.RECORD) { - return; - } - - AirbyteRecordMessage recordMessage = airbyteMessage.getRecord(); - AirbyteStreamNameNamespacePair pair = AirbyteStreamNameNamespacePair - .fromRecordMessage(recordMessage); - - if (!loggers.containsKey(pair)) { - throw new IllegalArgumentException( - String.format( - "Message contained record from a stream that was not in the catalog. \ncatalog: %s , \nmessage: %s", - Jsons.serialize(configuredCatalog), Jsons.serialize(recordMessage))); - } - - loggers.get(pair).log(recordMessage); - } - - @Override - protected void close(boolean hasFailed) { - if (!hasFailed) { - outputRecordCollector.accept(lastStateMessage); - } - } - -} diff --git a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/NullDestination.java b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/NullDestination.java deleted file mode 100644 index 13fca00407e4..000000000000 --- a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/NullDestination.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2021 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.destination_null; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.destination.destination_null.logger.NullDestinationLogger; -import io.airbyte.integrations.destination.destination_null.logger.NullDestinationLoggerFactory; -import io.airbyte.protocol.models.AirbyteConnectionStatus; -import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; -import io.airbyte.protocol.models.AirbyteMessage; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import java.util.function.Consumer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class NullDestination extends BaseConnector implements Destination { - - public static void main(String[] args) throws Exception { - new IntegrationRunner(new NullDestination()).run(args); - } - - /** - * The null destination is always available! - */ - @Override - public AirbyteConnectionStatus check(JsonNode config) { - return new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED); - } - - @Override - public AirbyteMessageConsumer getConsumer(JsonNode config, - ConfiguredAirbyteCatalog configuredCatalog, - Consumer outputRecordCollector) { - return new NullConsumer(new NullDestinationLoggerFactory(config), configuredCatalog, outputRecordCollector); - } - -} diff --git a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/BaseLogger.java b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/BaseLogger.java deleted file mode 100644 index df65e9f2be40..000000000000 --- a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/BaseLogger.java +++ /dev/null @@ -1,45 +0,0 @@ -package io.airbyte.integrations.destination.destination_null.logger; - -import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; -import io.airbyte.protocol.models.AirbyteRecordMessage; -import java.time.Instant; -import java.time.OffsetDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public abstract class BaseLogger implements NullDestinationLogger { - - protected final AirbyteStreamNameNamespacePair streamNamePair; - protected final int maxEntryCount; - protected int loggedEntryCount = 0; - - public BaseLogger(AirbyteStreamNameNamespacePair streamNamePair, int maxEntryCount) { - this.streamNamePair = streamNamePair; - this.maxEntryCount = maxEntryCount; - } - - protected String entryMessage(AirbyteRecordMessage recordMessage) { - return String.format("[%s] %s #%04d: %s", - emissionTimestamp(recordMessage.getEmittedAt()), - streamName(streamNamePair), - loggedEntryCount, - recordMessage.getData()); - } - - protected static String streamName(AirbyteStreamNameNamespacePair pair) { - if (pair.getNamespace() == null) { - return pair.getName(); - } else { - return String.format("%s.%s", pair.getNamespace(), pair.getName()); - } - } - - protected static String emissionTimestamp(long emittedAt) { - return OffsetDateTime - .ofInstant(Instant.ofEpochMilli(emittedAt), ZoneId.systemDefault()) - .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); - } - -} diff --git a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/DevNull.java b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/DevNull.java deleted file mode 100644 index a627cfa44f59..000000000000 --- a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/DevNull.java +++ /dev/null @@ -1,17 +0,0 @@ -package io.airbyte.integrations.destination.destination_null.logger; - -import io.airbyte.protocol.models.AirbyteRecordMessage; - -public class DevNull implements NullDestinationLogger { - - public static final DevNull SINGLETON = new DevNull(); - - private DevNull() { - } - - @Override - public void log(AirbyteRecordMessage recordMessage) { - // nothing happens in /dev/null - } - -} diff --git a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/EveryNthLogger.java b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/EveryNthLogger.java deleted file mode 100644 index 5d645e888dac..000000000000 --- a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/EveryNthLogger.java +++ /dev/null @@ -1,33 +0,0 @@ -package io.airbyte.integrations.destination.destination_null.logger; - -import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; -import io.airbyte.protocol.models.AirbyteRecordMessage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class EveryNthLogger extends BaseLogger implements NullDestinationLogger { - - private static final Logger LOGGER = LoggerFactory.getLogger(EveryNthLogger.class); - - private final int nthEntryToLog; - private int currentEntry = 0; - - public EveryNthLogger(AirbyteStreamNameNamespacePair streamNamePair, int nthEntryToLog, int maxEntryCount) { - super(streamNamePair, maxEntryCount); - this.nthEntryToLog = nthEntryToLog; - } - - @Override - public void log(AirbyteRecordMessage recordMessage) { - if (loggedEntryCount >= maxEntryCount) { - return; - } - - currentEntry += 1; - if (currentEntry % nthEntryToLog == 0) { - loggedEntryCount += 1; - LOGGER.info(entryMessage(recordMessage)); - } - } - -} diff --git a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/FirstNLogger.java b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/FirstNLogger.java deleted file mode 100644 index a394c657b479..000000000000 --- a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/FirstNLogger.java +++ /dev/null @@ -1,26 +0,0 @@ -package io.airbyte.integrations.destination.destination_null.logger; - -import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; -import io.airbyte.protocol.models.AirbyteRecordMessage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class FirstNLogger extends BaseLogger implements NullDestinationLogger { - - private static final Logger LOGGER = LoggerFactory.getLogger(FirstNLogger.class); - - public FirstNLogger(AirbyteStreamNameNamespacePair streamNamePair, int maxEntryCount) { - super(streamNamePair, maxEntryCount); - } - - @Override - public void log(AirbyteRecordMessage recordMessage) { - if (loggedEntryCount >= maxEntryCount) { - return; - } - - loggedEntryCount += 1; - LOGGER.info(entryMessage(recordMessage)); - } - -} diff --git a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/NullDestinationLogger.java b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/NullDestinationLogger.java deleted file mode 100644 index 627ab41b46a0..000000000000 --- a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/NullDestinationLogger.java +++ /dev/null @@ -1,16 +0,0 @@ -package io.airbyte.integrations.destination.destination_null.logger; - -import io.airbyte.protocol.models.AirbyteRecordMessage; - -public interface NullDestinationLogger { - - enum LoggingType { - NoLogging, - FirstN, - EveryNth, - RandomSampling - } - - void log(AirbyteRecordMessage recordMessage); - -} diff --git a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/NullDestinationLoggerFactory.java b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/NullDestinationLoggerFactory.java deleted file mode 100644 index 0818264b32f8..000000000000 --- a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/NullDestinationLoggerFactory.java +++ /dev/null @@ -1,45 +0,0 @@ -package io.airbyte.integrations.destination.destination_null.logger; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; -import io.airbyte.integrations.destination.destination_null.logger.NullDestinationLogger.LoggingType; - -public class NullDestinationLoggerFactory { - - private final JsonNode config; - - public NullDestinationLoggerFactory(JsonNode config) { - this.config = config; - } - - public NullDestinationLogger create(AirbyteStreamNameNamespacePair streamNamePair) { - if (!config.has("logging")) { - throw new IllegalArgumentException("Property logging is required, but not found"); - } - - final JsonNode logConfig = config.get("logging"); - final LoggingType loggingType = LoggingType.valueOf(logConfig.get("logging_type").asText()); - switch (loggingType) { - case NoLogging -> { - return DevNull.SINGLETON; - } - case FirstN -> { - final int maxEntryCount = logConfig.get("max_entry_count").asInt(); - return new FirstNLogger(streamNamePair, maxEntryCount); - } - case EveryNth -> { - final int nthEntryToLog = logConfig.get("nth_entry_to_log").asInt(); - final int maxEntryCount = logConfig.get("max_entry_count").asInt(); - return new EveryNthLogger(streamNamePair, nthEntryToLog, maxEntryCount); - } - case RandomSampling -> { - final double samplingRatio = logConfig.get("sampling_ratio").asDouble(); - final long seed = logConfig.has("seed") ? logConfig.get("seed").asLong() : System.currentTimeMillis(); - final int maxEntryCount = logConfig.get("max_entry_count").asInt(); - return new RandomSamplingLogger(streamNamePair, samplingRatio, seed, maxEntryCount); - } - default -> throw new IllegalArgumentException("Unexpected logging type: " + loggingType); - } - } - -} diff --git a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/RandomSamplingLogger.java b/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/RandomSamplingLogger.java deleted file mode 100644 index aecb98d4f03d..000000000000 --- a/airbyte-integrations/connectors/destination-null/src/main/java/io/airbyte/integrations/destination/destination_null/logger/RandomSamplingLogger.java +++ /dev/null @@ -1,34 +0,0 @@ -package io.airbyte.integrations.destination.destination_null.logger; - -import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; -import io.airbyte.protocol.models.AirbyteRecordMessage; -import java.util.Random; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class RandomSamplingLogger extends BaseLogger implements NullDestinationLogger { - - private static final Logger LOGGER = LoggerFactory.getLogger(RandomSamplingLogger.class); - - private final double samplingRatio; - private final Random random; - - public RandomSamplingLogger(AirbyteStreamNameNamespacePair streamNamePair, double samplingRatio, long seed, int maxEntryCount) { - super(streamNamePair, maxEntryCount); - this.samplingRatio = samplingRatio; - this.random = new Random(seed); - } - - @Override - public void log(AirbyteRecordMessage recordMessage) { - if (loggedEntryCount >= maxEntryCount) { - return; - } - - if (random.nextDouble() < samplingRatio) { - loggedEntryCount += 1; - LOGGER.info(entryMessage(recordMessage)); - } - } - -} diff --git a/airbyte-integrations/connectors/destination-null/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-null/src/main/resources/spec.json deleted file mode 100644 index 55c96bba3a3e..000000000000 --- a/airbyte-integrations/connectors/destination-null/src/main/resources/spec.json +++ /dev/null @@ -1,123 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/null", - "supportsIncremental": true, - "supported_destination_sync_modes": ["overwrite", "append"], - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Null Destination Spec", - "type": "object", - "required": ["logging"], - "additionalProperties": false, - "properties": { - "logging": { - "title": "Log Output", - "type": "object", - "description": "Whether the data should be logged.", - "oneOf": [ - { - "title": "No Logging", - "description": "Log nothing. A pure /dev/null.", - "type": "object", - "required": ["logging_type"], - "properties": { - "logging_type": { - "type": "string", - "enum": ["NoLogging"], - "default": "NoLogging" - } - } - }, - { - "title": "First N Entries", - "description": "Log first N entries per stream.", - "type": "object", - "required": ["logging_type", "max_entry_count"], - "properties": { - "logging_type": { - "type": "string", - "enum": ["FirstN"], - "default": "FirstN" - }, - "max_entry_count": { - "title": "N", - "description": "Number of entries to log. This destination is for testing only. So it won't make sense to log infinitely. The maximum is 1,000 entries.", - "type": "number", - "default": 100, - "examples": [100], - "minimum": 1, - "maximum": 1000 - } - } - }, - { - "title": "Every N-th Entry", - "description": "For each stream, log every N-th entry with a maximum cap.", - "type": "object", - "required": ["logging_type", "nth_entry_to_log", "max_entry_count"], - "properties": { - "logging_type": { - "type": "string", - "enum": ["EveryNth"], - "default": "EveryNth" - }, - "nth_entry_to_log": { - "title": "N", - "description": "The N-th entry to log for each stream. N starts from 1. For example, when N = 1, every entry is logged; when N = 2, every other entry is logged; when N = 3, one out of three entries is logged.", - "type": "number", - "example": [3], - "minimum": 1, - "maximum": 1000 - }, - "max_entry_count": { - "title": "Max Log Entries", - "description": "Max number of entries to log. This destination is for testing only. So it won't make sense to log infinitely. The maximum is 1,000 entries.", - "type": "number", - "default": 100, - "examples": [100], - "minimum": 1, - "maximum": 1000 - } - } - }, - { - "title": "Random Sampling", - "description": "For each stream, randomly log a percentage of the entries with a maximum cap.", - "type": "object", - "required": ["logging_type", "sampling_ratio", "max_entry_count"], - "properties": { - "logging_type": { - "type": "string", - "enum": ["RandomSampling"], - "default": "RandomSampling" - }, - "sampling_ratio": { - "title": "Sampling Ratio", - "description": "A positive floating number smaller than 1.", - "type": "number", - "default": 0.001, - "examples": [0.001], - "minimum": 0, - "maximum": 1 - }, - "seed": { - "title": "Random Number Generator Seed", - "description": "When the seed is unspecified, the current time millis will be used as the seed.", - "type": "number", - "examples": [1900] - }, - "max_entry_count": { - "title": "Max Log Entries", - "description": "Max number of entries to log. This destination is for testing only. So it won't make sense to log infinitely. The maximum is 1,000 entries.", - "type": "number", - "default": 100, - "examples": [100], - "minimum": 1, - "maximum": 1000 - } - } - } - ] - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-null/src/test-integration/java/io/airbyte/integrations/destination/destination_null/NullDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-null/src/test-integration/java/io/airbyte/integrations/destination/destination_null/NullDestinationAcceptanceTest.java deleted file mode 100644 index 6a1263b42573..000000000000 --- a/airbyte-integrations/connectors/destination-null/src/test-integration/java/io/airbyte/integrations/destination/destination_null/NullDestinationAcceptanceTest.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2021 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.destination_null; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.protocol.models.AirbyteMessage; -import io.airbyte.protocol.models.AirbyteRecordMessage; -import java.util.Collections; -import java.util.List; - -public class NullDestinationAcceptanceTest extends DestinationAcceptanceTest { - - @Override - protected String getImageName() { - return "airbyte/destination-null:dev"; - } - - @Override - protected JsonNode getConfig() { - final JsonNode loggingConfig = Jsons.jsonNode(ImmutableMap.builder() - .put("logging_type", "EveryNth") - .put("nth_entry_to_log", 1) - .put("max_entry_count", 3) - .build()); - return Jsons.jsonNode(Collections.singletonMap("logging", loggingConfig)); - } - - @Override - protected JsonNode getFailCheckConfig() { - final JsonNode loggingConfig = Jsons.jsonNode(ImmutableMap.builder() - .put("logging_type", "LastN") - .put("max_entry_count", 3000) - .build()); - return Jsons.jsonNode(Collections.singletonMap("logging", loggingConfig)); - } - - @Override - protected List retrieveRecords(TestDestinationEnv testEnv, - String streamName, - String namespace, - JsonNode streamSchema) { - return Collections.emptyList(); - } - - @Override - protected void setup(TestDestinationEnv testEnv) { - // do nothing - } - - @Override - protected void tearDown(TestDestinationEnv testEnv) { - // do nothing - } - - @Override - protected void assertSameMessages(List expected, List actual, boolean pruneAirbyteInternalFields) { - // do nothing - } - -} From c5f0de9338a4590fce3fce6fa5a8ca4657c29381 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Thu, 16 Dec 2021 22:49:05 -0800 Subject: [PATCH 05/17] Add documentation --- .../destination-e2e-test/Dockerfile | 2 +- .../connectors/destination-e2e-test/README.md | 67 +++++++++++++++++++ .../destination-e2e-test/build.gradle | 16 ++--- docs/integrations/destinations/e2e-test.md | 46 +++++++++++++ 4 files changed, 118 insertions(+), 13 deletions(-) create mode 100644 airbyte-integrations/connectors/destination-e2e-test/README.md create mode 100644 docs/integrations/destinations/e2e-test.md diff --git a/airbyte-integrations/connectors/destination-e2e-test/Dockerfile b/airbyte-integrations/connectors/destination-e2e-test/Dockerfile index 76348c3209ca..01cea29bc52b 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/Dockerfile +++ b/airbyte-integrations/connectors/destination-e2e-test/Dockerfile @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/destination-e2e-test diff --git a/airbyte-integrations/connectors/destination-e2e-test/README.md b/airbyte-integrations/connectors/destination-e2e-test/README.md new file mode 100644 index 000000000000..43e0d5bc6720 --- /dev/null +++ b/airbyte-integrations/connectors/destination-e2e-test/README.md @@ -0,0 +1,67 @@ +# End-to-End Testing Destination + +This is the repository for the Null destination connector in Java. For information about how to use this connector within Airbyte, see [the User Documentation](https://docs.airbyte.io/integrations/destinations/e2e-test). + +## Local development + +#### Building via Gradle +From the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:destination-:build +``` + +#### Create credentials +**If you are a community contributor**, generate the necessary credentials and place them in `secrets/config.json` conforming to the spec file in `src/main/resources/spec.json`. +Note that the `secrets` directory is git-ignored by default, so there is no danger of accidentally checking in sensitive information. + +**If you are an Airbyte core member**, follow the [instructions](https://docs.airbyte.io/connector-development#using-credentials-in-ci) to set up the credentials. + +### Locally running the connector docker image + +#### Build +Build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:destination-e2e-test:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/destination-e2e-test:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-e2e-test:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-e2e-test:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-e2e-test:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +## Testing +We use `JUnit` for Java tests. + +### Unit and Integration Tests +Place unit tests under `src/test/io/airbyte/integrations/destinations/e2e-test`. + +#### Acceptance Tests +Airbyte has a standard test suite that all destination connectors must pass. See example(s) in +`src/test-integration/java/io/airbyte/integrations/destinations/e2e-test/`. + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:destination-e2e-test:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:destination-e2e-test:integrationTest +``` + +## Dependency Management + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +2. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +3. Create a Pull Request. +4. Pat yourself on the back for being an awesome contributor. +5. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/destination-e2e-test/build.gradle b/airbyte-integrations/connectors/destination-e2e-test/build.gradle index fbebb2dec265..4ff3a27be3d8 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/build.gradle +++ b/airbyte-integrations/connectors/destination-e2e-test/build.gradle @@ -9,19 +9,11 @@ application { } dependencies { - implementation project(':airbyte-db:lib') - implementation project(':airbyte-integrations:bases:base-java') + implementation project(':airbyte-config:models') implementation project(':airbyte-protocol:models') - implementation project(':airbyte-integrations:connectors:destination-jdbc') - - testImplementation project(':airbyte-test-utils') - - testImplementation "org.testcontainers:postgresql:1.15.3" + implementation project(':airbyte-integrations:bases:base-java') + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - - integrationTestJavaImplementation "org.testcontainers:postgresql:1.15.3" - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) + integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-e2e-test') } diff --git a/docs/integrations/destinations/e2e-test.md b/docs/integrations/destinations/e2e-test.md new file mode 100644 index 000000000000..e78f3327c572 --- /dev/null +++ b/docs/integrations/destinations/e2e-test.md @@ -0,0 +1,46 @@ +# End-to-End Testing Destination + +This destination is for testing of Airbyte connections. It can be set up as a source message logger, a `/dev/null`, or to mimic specific behaviors (e.g. exception during the sync). Please use it with discretion. This destination may log your data, and expose sensitive information. + +## Features + +| Feature | Supported | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | Yes | | +| Incremental Sync | No | | +| Replicate Incremental Deletes | No | | +| SSL connection | No | | +| SSH Tunnel Support | No | | + +## Mode + +### Logging + +This mode logs the data from the source connector. It will log at most 1,000 data entries. + +There are the different logging modes to choose from: + +| Mode | Notes | Parameters | +| :--- | :--- | :--- | +| First N entries | Log the first N number of data entries for each data stream. | N: how many entries to log. | +| Every N-th entry | Log every N-th entry for each data stream. When N=1, it will log every entry. When N=1, it will log every other entry. Etc. | N: the N-th entry to log. Max entry count: max number of entries to log. | +| Random sampling | Log a random percentage of the entries for each data stream. | Sampling ratio: a number in range of `[0, 1]`. Optional seed: default to system epoch time. Max entry count: max number of entries to log. | + +### `/dev/null` + +This mode works as `/dev/null`. It does nothing about any data from the source connector. This is usually only useful for performance testing of the source connector. + +### Throttling + +This mode mimics a slow data sync. You can specify the time (in millisecond) of delay between each message from the source is processed. + +### Failing + +This mode throws an exception after receiving a configurable number of messages. + +## CHANGELOG + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :--- | +| 0.2.0 | 2021-12-16 | [\#8824](https://github.com/airbytehq/airbyte/pull/8824) | Add multiple logging modes. | +| 0.1.0 | 2021-05-25 | [\#3290](https://github.com/airbytehq/airbyte/pull/3290) | Create initial version. | From 431dad0ddde257e15eb6415a43e401fdf1e84ae9 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Thu, 16 Dec 2021 22:53:35 -0800 Subject: [PATCH 06/17] Add destination to build and summary --- airbyte-integrations/builds.md | 1 + docs/SUMMARY.md | 1 + 2 files changed, 2 insertions(+) diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index e204cf7fb5ac..71b56c31c76e 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -113,6 +113,7 @@ | Cassandra | [![destination-cassandra](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-cassandra%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-cassandra) | | Databricks | (Temporarily Not Available) | | Elasticsearch | (Temporarily Not Available) | +| End-to-End Testing | [![destination-e2e-test](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-e2e-test%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-e2e-test) | | Google Cloud Storage (GCS) | [![destination-gcs](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-gcs%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-gcs) | | Google Firestore | [![destination-firestore](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-firestore%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-firestore) | | Google PubSub | [![destination-pubsub](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-pubsub%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-pubsub) | diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index c237c61a1cba..7d0255eb5d33 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -164,6 +164,7 @@ * [Databricks](integrations/destinations/databricks.md) * [DynamoDB](integrations/destinations/dynamodb.md) * [Elasticsearch](integrations/destinations/elasticsearch.md) + * [End-to-End Testing](integrations/destinations/e2e-test.md) * [Chargify](integrations/destinations/chargify.md) * [Google Cloud Storage (GCS)](integrations/destinations/gcs.md) * [Google Firestore](integrations/destinations/firestore.md) From 35200bcd9509d1fd54dd58500bddd02c2efa86d8 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Thu, 16 Dec 2021 23:21:11 -0800 Subject: [PATCH 07/17] Fix test --- .../e2e_test/TestingDestinations.java | 7 +++++- .../src/main/resources/spec.json | 4 ---- ...stingSilentDestinationAcceptanceTest.java} | 22 +++++-------------- docs/integrations/destinations/e2e-test.md | 2 +- 4 files changed, 13 insertions(+), 22 deletions(-) rename airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/{TestingDestinationLoggingEveryNthAcceptanceTest.java => TestingSilentDestinationAcceptanceTest.java} (64%) diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/TestingDestinations.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/TestingDestinations.java index c6bf3ee08dbe..a9055b8ba173 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/TestingDestinations.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/TestingDestinations.java @@ -11,6 +11,7 @@ import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.protocol.models.AirbyteConnectionStatus; +import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import java.util.Map; @@ -58,7 +59,11 @@ public AirbyteMessageConsumer getConsumer(final JsonNode config, @Override public AirbyteConnectionStatus check(final JsonNode config) throws Exception { - return selectDestination(config).check(config); + try { + return selectDestination(config).check(config); + } catch (final Exception e) { + return new AirbyteConnectionStatus().withStatus(Status.FAILED).withMessage(e.getMessage()); + } } public static void main(final String[] args) throws Exception { diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json index 2c0b9d83e3a7..bfdcb6a75e7b 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json @@ -12,7 +12,6 @@ { "title": "Logging", "required": ["type", "logging_config"], - "additionalProperties": false, "properties": { "type": { "type": "string", @@ -120,7 +119,6 @@ { "title": "Silent", "required": ["type"], - "additionalProperties": false, "properties": { "type": { "type": "string", @@ -132,7 +130,6 @@ { "title": "Throttled", "required": ["type", "millis_per_record"], - "additionalProperties": false, "properties": { "type": { "type": "string", @@ -148,7 +145,6 @@ { "title": "Failing", "required": ["type", "num_messages"], - "additionalProperties": false, "properties": { "type": { "type": "string", diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingDestinationLoggingEveryNthAcceptanceTest.java b/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.java similarity index 64% rename from airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingDestinationLoggingEveryNthAcceptanceTest.java rename to airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.java index 3a3c9dba6c71..19ed30b249e0 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingDestinationLoggingEveryNthAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.java @@ -1,7 +1,8 @@ package io.airbyte.integrations.destination.e2e_test; +import static org.junit.jupiter.api.Assertions.assertEquals; + import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.destination.e2e_test.TestingDestinations.TestDestinationType; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; @@ -9,9 +10,8 @@ import io.airbyte.protocol.models.AirbyteRecordMessage; import java.util.Collections; import java.util.List; -import java.util.Map; -public class TestingDestinationLoggingEveryNthAcceptanceTest extends DestinationAcceptanceTest { +public class TestingSilentDestinationAcceptanceTest extends DestinationAcceptanceTest { @Override protected String getImageName() { @@ -20,22 +20,12 @@ protected String getImageName() { @Override protected JsonNode getConfig() { - final JsonNode loggingConfig = Jsons.jsonNode(ImmutableMap.builder() - .put("logging_type", "EveryNth") - .put("nth_entry_to_log", 1) - .put("max_entry_count", 3) - .build()); - return Jsons.jsonNode(Map.of("type", TestDestinationType.LOGGING.name(), "logging_config", loggingConfig)); + return Jsons.jsonNode(Collections.singletonMap("type", TestDestinationType.SILENT.name())); } @Override protected JsonNode getFailCheckConfig() { - final JsonNode loggingConfig = Jsons.jsonNode(ImmutableMap.builder() - .put("logging_type", "LastN") - // max allowed entry count is 1000 - .put("max_entry_count", 3000) - .build()); - return Jsons.jsonNode(Map.of("type", TestDestinationType.LOGGING.name(), "logging_config", loggingConfig)); + return Jsons.jsonNode(Collections.singletonMap("type", "invalid")); } @Override @@ -60,7 +50,7 @@ protected void tearDown(final TestDestinationEnv testEnv) { protected void assertSameMessages(final List expected, final List actual, final boolean pruneAirbyteInternalFields) { - // do nothing + assertEquals(0, actual.size()); } } diff --git a/docs/integrations/destinations/e2e-test.md b/docs/integrations/destinations/e2e-test.md index e78f3327c572..064174667af2 100644 --- a/docs/integrations/destinations/e2e-test.md +++ b/docs/integrations/destinations/e2e-test.md @@ -7,7 +7,7 @@ This destination is for testing of Airbyte connections. It can be set up as a so | Feature | Supported | Notes | | :--- | :--- | :--- | | Full Refresh Sync | Yes | | -| Incremental Sync | No | | +| Incremental Sync | Yes | | | Replicate Incremental Deletes | No | | | SSL connection | No | | | SSH Tunnel Support | No | | From 31690b2979b682e46195fb7979cf3fd77086aa3c Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Thu, 16 Dec 2021 23:23:32 -0800 Subject: [PATCH 08/17] Update acceptance test --- .../java/io/airbyte/test/acceptance/AcceptanceTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java index 0a574582293a..07eef7adcfb6 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java @@ -719,7 +719,7 @@ public void testCheckpointing() throws Exception { "E2E Test Destination -" + UUID.randomUUID(), workspaceId, destinationDefinition.getDestinationDefinitionId(), - Jsons.jsonNode(ImmutableMap.of("type", "LOGGING"))); + Jsons.jsonNode(ImmutableMap.of("type", "SILENT"))); final String connectionName = "test-connection"; final UUID sourceId = source.getSourceId(); From ae075ce0ae2468fed5b0f29bbd56929bff322819 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Thu, 16 Dec 2021 23:35:14 -0800 Subject: [PATCH 09/17] Log state message --- .../e2e_test/logging/LoggingConsumer.java | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.java index 4fb3870ae548..5ff6f3244990 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.java @@ -1,8 +1,8 @@ package io.airbyte.integrations.destination.e2e_test.logging; import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.AirbyteMessageConsumer; import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.AirbyteRecordMessage; @@ -12,19 +12,23 @@ import java.util.HashMap; import java.util.Map; import java.util.function.Consumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class LoggingConsumer extends FailureTrackingAirbyteMessageConsumer { +public class LoggingConsumer implements AirbyteMessageConsumer { + + private static final Logger LOGGER = LoggerFactory.getLogger(LoggingConsumer.class); private final TestingLoggerFactory loggerFactory; private final ConfiguredAirbyteCatalog configuredCatalog; private final Consumer outputRecordCollector; private final Map loggers; - private AirbyteMessage lastStateMessage = null; + private final AirbyteMessage lastStateMessage = null; public LoggingConsumer(final TestingLoggerFactory loggerFactory, - final ConfiguredAirbyteCatalog configuredCatalog, - final Consumer outputRecordCollector) { + final ConfiguredAirbyteCatalog configuredCatalog, + final Consumer outputRecordCollector) { this.loggerFactory = loggerFactory; this.configuredCatalog = configuredCatalog; this.outputRecordCollector = outputRecordCollector; @@ -32,7 +36,7 @@ public LoggingConsumer(final TestingLoggerFactory loggerFactory, } @Override - protected void startTracked() { + public void start() { for (final ConfiguredAirbyteStream configuredStream : configuredCatalog.getStreams()) { final AirbyteStream stream = configuredStream.getStream(); final AirbyteStreamNameNamespacePair streamNamePair = AirbyteStreamNameNamespacePair.fromAirbyteSteam(stream); @@ -42,22 +46,22 @@ protected void startTracked() { } @Override - protected void acceptTracked(final AirbyteMessage airbyteMessage) { - if (airbyteMessage.getType() == Type.STATE) { - this.lastStateMessage = airbyteMessage; + public void accept(final AirbyteMessage message) { + if (message.getType() == Type.STATE) { + LOGGER.info("Emitting state: {}", message); + outputRecordCollector.accept(message); return; - } else if (airbyteMessage.getType() != Type.RECORD) { + } else if (message.getType() != Type.RECORD) { return; } - final AirbyteRecordMessage recordMessage = airbyteMessage.getRecord(); - final AirbyteStreamNameNamespacePair pair = AirbyteStreamNameNamespacePair - .fromRecordMessage(recordMessage); + final AirbyteRecordMessage recordMessage = message.getRecord(); + final AirbyteStreamNameNamespacePair pair = AirbyteStreamNameNamespacePair.fromRecordMessage(recordMessage); if (!loggers.containsKey(pair)) { throw new IllegalArgumentException( String.format( - "Message contained record from a stream that was not in the catalog. \ncatalog: %s , \nmessage: %s", + "Message contained record from a stream that was not in the catalog.\n Catalog: %s\n Message: %s", Jsons.serialize(configuredCatalog), Jsons.serialize(recordMessage))); } @@ -65,10 +69,7 @@ protected void acceptTracked(final AirbyteMessage airbyteMessage) { } @Override - protected void close(final boolean hasFailed) { - if (!hasFailed) { - outputRecordCollector.accept(lastStateMessage); - } + public void close() { } } From f39206f7d10bd0e692403c32326921dd9e3c14d7 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Thu, 16 Dec 2021 23:37:21 -0800 Subject: [PATCH 10/17] Remove unused variable --- .../destination/e2e_test/logging/LoggingConsumer.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.java index 5ff6f3244990..3ef386f10108 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.java @@ -24,8 +24,6 @@ public class LoggingConsumer implements AirbyteMessageConsumer { private final Consumer outputRecordCollector; private final Map loggers; - private final AirbyteMessage lastStateMessage = null; - public LoggingConsumer(final TestingLoggerFactory loggerFactory, final ConfiguredAirbyteCatalog configuredCatalog, final Consumer outputRecordCollector) { From 6a67aac2557ef7c253d002424a6b6999cc0b3ca8 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Thu, 16 Dec 2021 23:41:18 -0800 Subject: [PATCH 11/17] Remove extra statement --- airbyte-integrations/connectors/destination-e2e-test/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-e2e-test/Dockerfile b/airbyte-integrations/connectors/destination-e2e-test/Dockerfile index e5781a44b194..0b6998be575f 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/Dockerfile +++ b/airbyte-integrations/connectors/destination-e2e-test/Dockerfile @@ -6,7 +6,5 @@ ENV APPLICATION destination-e2e-test ADD build/distributions/${APPLICATION}*.tar /airbyte -RUN tar xf ${APPLICATION}.tar --strip-components=1 - LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/destination-e2e-test From 7536f53235848dadd5cf7a822cfb6888ad7edc84 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Sat, 18 Dec 2021 14:27:39 -0800 Subject: [PATCH 12/17] Remove old null doc --- docs/integrations/destinations/null.md | 52 -------------------------- 1 file changed, 52 deletions(-) delete mode 100644 docs/integrations/destinations/null.md diff --git a/docs/integrations/destinations/null.md b/docs/integrations/destinations/null.md deleted file mode 100644 index 966555a7c4bd..000000000000 --- a/docs/integrations/destinations/null.md +++ /dev/null @@ -1,52 +0,0 @@ -# Null - -TODO: update this doc - -## Sync overview - -### Output schema - -Is the output schema fixed (e.g: for an API like Stripe)? If so, point to the connector's schema (e.g: link to Stripe’s documentation) or describe the schema here directly (e.g: include a diagram or paragraphs describing the schema). - -Describe how the connector's schema is mapped to Airbyte concepts. An example description might be: "MagicDB tables become Airbyte Streams and MagicDB columns become Airbyte Fields. In addition, an extracted\_at column is appended to each row being read." - -### Data type mapping - -This section should contain a table mapping each of the connector's data types to Airbyte types. At the moment, Airbyte uses the same types used by [JSONSchema](https://json-schema.org/understanding-json-schema/reference/index.html). `string`, `date-time`, `object`, `array`, `boolean`, `integer`, and `number` are the most commonly used data types. - -| Integration Type | Airbyte Type | Notes | -| :--- | :--- | :--- | - - -### Features - -This section should contain a table with the following format: - -| Feature | Supported?(Yes/No) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | | | -| Incremental Sync | | | -| Replicate Incremental Deletes | | | -| For databases, WAL/Logical replication | | | -| SSL connection | | | -| SSH Tunnel Support | | | -| (Any other source-specific features) | | | - -### Performance considerations - -Could this connector hurt the user's database/API/etc... or put too much strain on it in certain circumstances? For example, if there are a lot of tables or rows in a table? What is the breaking point (e.g: 100mm> records)? What can the user do to prevent this? (e.g: use a read-only replica, or schedule frequent syncs, etc..) - -## Getting started - -### Requirements - -* What versions of this connector does this implementation support? (e.g: `postgres v3.14 and above`) -* What configurations, if any, are required on the connector? (e.g: `buffer_size > 1024`) -* Network accessibility requirements -* Credentials/authentication requirements? (e.g: A DB user with read permissions on certain tables) - -### Setup guide - -For each of the above high-level requirements as appropriate, add or point to a follow-along guide. See existing source or destination guides for an example. - -For each major cloud provider we support, also add a follow-along guide for setting up Airbyte to connect to that destination. See the Postgres destination guide for an example of what this should look like. From c966d44319bf321bf16006d126b637fc14b05668 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Sat, 18 Dec 2021 15:46:27 -0800 Subject: [PATCH 13/17] Add dev null destination --- airbyte-integrations/builds.md | 1 + .../destination-dev-null/.dockerignore | 3 + .../destination-dev-null/Dockerfile | 10 ++++ .../connectors/destination-dev-null/README.md | 51 ++++++++++++++++ .../destination-dev-null/build.gradle | 20 +++++++ .../dev_null/DevNullDestination.java | 56 ++++++++++++++++++ .../DevNullDestinationAcceptanceTest.java | 59 +++++++++++++++++++ .../dev_null/DevNullDestinationTest.java | 20 +++++++ .../src/test/resources/expected_spec.json | 25 ++++++++ .../destination-e2e-test/Dockerfile | 3 +- .../connectors/destination-e2e-test/README.md | 3 + docs/integrations/destinations/e2e-test.md | 10 ++-- 12 files changed, 256 insertions(+), 5 deletions(-) create mode 100644 airbyte-integrations/connectors/destination-dev-null/.dockerignore create mode 100644 airbyte-integrations/connectors/destination-dev-null/Dockerfile create mode 100644 airbyte-integrations/connectors/destination-dev-null/README.md create mode 100644 airbyte-integrations/connectors/destination-dev-null/build.gradle create mode 100644 airbyte-integrations/connectors/destination-dev-null/src/main/java/io/airbyte/integrations/destination/dev_null/DevNullDestination.java create mode 100644 airbyte-integrations/connectors/destination-dev-null/src/test-integration/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.java create mode 100644 airbyte-integrations/connectors/destination-dev-null/src/test/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationTest.java create mode 100644 airbyte-integrations/connectors/destination-dev-null/src/test/resources/expected_spec.json diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 71b56c31c76e..3ae161b2c5d3 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -112,6 +112,7 @@ | ClickHouse | [![destination-clickhouse](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-clickhouse%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-clickhouse) | | Cassandra | [![destination-cassandra](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-cassandra%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-cassandra) | | Databricks | (Temporarily Not Available) | +| Dev Null | [![destination-dev-null](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-dev-null%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-dev-null) | | Elasticsearch | (Temporarily Not Available) | | End-to-End Testing | [![destination-e2e-test](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-e2e-test%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-e2e-test) | | Google Cloud Storage (GCS) | [![destination-gcs](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-gcs%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-gcs) | diff --git a/airbyte-integrations/connectors/destination-dev-null/.dockerignore b/airbyte-integrations/connectors/destination-dev-null/.dockerignore new file mode 100644 index 000000000000..65c7d0ad3e73 --- /dev/null +++ b/airbyte-integrations/connectors/destination-dev-null/.dockerignore @@ -0,0 +1,3 @@ +* +!Dockerfile +!build diff --git a/airbyte-integrations/connectors/destination-dev-null/Dockerfile b/airbyte-integrations/connectors/destination-dev-null/Dockerfile new file mode 100644 index 000000000000..e66cddd27b4d --- /dev/null +++ b/airbyte-integrations/connectors/destination-dev-null/Dockerfile @@ -0,0 +1,10 @@ +FROM airbyte/integration-base-java:dev + +WORKDIR /airbyte +ENV APPLICATION destination-dev-null + +COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar +RUN tar xf ${APPLICATION}.tar --strip-components=1 + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/destination-dev-null diff --git a/airbyte-integrations/connectors/destination-dev-null/README.md b/airbyte-integrations/connectors/destination-dev-null/README.md new file mode 100644 index 000000000000..bfbf20a9cf2c --- /dev/null +++ b/airbyte-integrations/connectors/destination-dev-null/README.md @@ -0,0 +1,51 @@ +# Destination Dev Null + +This destination is a "safe" version of the [E2E Test destination](https://docs.airbyte.io/integrations/destinations/e2e-test). It only allows the "silent" mode. + +## Local development + +#### Building via Gradle +From the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:destination-dev-null:build +``` + +### Locally running the connector docker image + +#### Build +Build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:destination-dev-null:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/destination-dev-null:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-dev-null:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-dev-null:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-dev-null:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:destination-dev-null:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:destination-dev-null:integrationTest +``` + +## Dependency Management + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/destination-dev-null/build.gradle b/airbyte-integrations/connectors/destination-dev-null/build.gradle new file mode 100644 index 000000000000..9e8e9eaf9e44 --- /dev/null +++ b/airbyte-integrations/connectors/destination-dev-null/build.gradle @@ -0,0 +1,20 @@ +plugins { + id 'application' + id 'airbyte-docker' + id 'airbyte-integration-test-java' +} + +application { + mainClass = 'io.airbyte.integrations.destination.dev_null.DevNullDestination' +} + +dependencies { + implementation project(':airbyte-config:models') + implementation project(':airbyte-protocol:models') + implementation project(':airbyte-integrations:bases:base-java') + implementation project(':airbyte-integrations:connectors:destination-e2e-test') + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + + integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') + integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-dev-null') +} diff --git a/airbyte-integrations/connectors/destination-dev-null/src/main/java/io/airbyte/integrations/destination/dev_null/DevNullDestination.java b/airbyte-integrations/connectors/destination-dev-null/src/main/java/io/airbyte/integrations/destination/dev_null/DevNullDestination.java new file mode 100644 index 000000000000..3e3f0aff6a59 --- /dev/null +++ b/airbyte-integrations/connectors/destination-dev-null/src/main/java/io/airbyte/integrations/destination/dev_null/DevNullDestination.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.dev_null; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.Destination; +import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.integrations.base.spec_modification.SpecModifyingDestination; +import io.airbyte.integrations.destination.e2e_test.TestingDestinations; +import io.airbyte.protocol.models.ConnectorSpecification; +import java.util.Iterator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DevNullDestination extends SpecModifyingDestination implements Destination { + + private static final Logger LOGGER = LoggerFactory.getLogger(DevNullDestination.class); + private static final String DEV_NULL_DESTINATION_TITLE = "Dev Null Destination Spec"; + + public DevNullDestination() { + super(new TestingDestinations()); + } + + public static void main(final String[] args) throws Exception { + LOGGER.info("Starting destination: {}", DevNullDestination.class); + new IntegrationRunner(new DevNullDestination()).run(args); + LOGGER.info("Completed destination: {}", DevNullDestination.class); + } + + /** + * 1. Update the title. + * 2. Only keep the "silent" mode. + */ + @Override + public ConnectorSpecification modifySpec(final ConnectorSpecification originalSpec) { + final ConnectorSpecification spec = Jsons.clone(originalSpec); + + ((ObjectNode) spec.getConnectionSpecification()).put("title", DEV_NULL_DESTINATION_TITLE); + + final ArrayNode types = (ArrayNode) spec.getConnectionSpecification().get("oneOf"); + final Iterator typesIterator = types.elements(); + while (typesIterator.hasNext()) { + final JsonNode typeNode = typesIterator.next(); + if (!typeNode.get("properties").get("type").get("const").asText().equalsIgnoreCase("silent")) { + typesIterator.remove(); + } + } + return spec; + } + +} diff --git a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.java new file mode 100644 index 000000000000..2946b692f022 --- /dev/null +++ b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.dev_null; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import java.util.Collections; +import java.util.List; + +public class DevNullDestinationAcceptanceTest extends DestinationAcceptanceTest { + + @Override + protected String getImageName() { + return "airbyte/destination-dev-null:dev"; + } + + @Override + protected JsonNode getConfig() { + return Jsons.jsonNode(Collections.singletonMap("type", "SILENT")); + } + + @Override + protected JsonNode getFailCheckConfig() { + return Jsons.jsonNode(Collections.singletonMap("type", "invalid")); + } + + @Override + protected List retrieveRecords(final TestDestinationEnv testEnv, + final String streamName, + final String namespace, + final JsonNode streamSchema) { + return Collections.emptyList(); + } + + @Override + protected void setup(final TestDestinationEnv testEnv) { + // do nothing + } + + @Override + protected void tearDown(final TestDestinationEnv testEnv) { + // do nothing + } + + @Override + protected void assertSameMessages(final List expected, + final List actual, + final boolean pruneAirbyteInternalFields) { + assertEquals(0, actual.size()); + } + +} diff --git a/airbyte-integrations/connectors/destination-dev-null/src/test/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationTest.java b/airbyte-integrations/connectors/destination-dev-null/src/test/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationTest.java new file mode 100644 index 000000000000..391fbbaeab9f --- /dev/null +++ b/airbyte-integrations/connectors/destination-dev-null/src/test/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationTest.java @@ -0,0 +1,20 @@ +package io.airbyte.integrations.destination.dev_null; + +import static org.junit.jupiter.api.Assertions.*; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.protocol.models.ConnectorSpecification; +import org.junit.jupiter.api.Test; + +class DevNullDestinationTest { + + @Test + public void testSpec() throws Exception { + final ConnectorSpecification actual = new DevNullDestination().spec(); + final ConnectorSpecification expected = Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class); + + assertEquals(expected, actual); + } + +} diff --git a/airbyte-integrations/connectors/destination-dev-null/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/destination-dev-null/src/test/resources/expected_spec.json new file mode 100644 index 000000000000..f162bc42884a --- /dev/null +++ b/airbyte-integrations/connectors/destination-dev-null/src/test/resources/expected_spec.json @@ -0,0 +1,25 @@ +{ + "documentationUrl": "https://example.com", + "supportsIncremental": true, + "supportsNormalization": false, + "supportsDBT": false, + "supported_destination_sync_modes": ["overwrite", "append"], + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Dev Null Destination Spec", + "type": "object", + "oneOf": [ + { + "title": "Silent", + "required": ["type"], + "properties": { + "type": { + "type": "string", + "const": "SILENT", + "default": "SILENT" + } + } + } + ] + } +} diff --git a/airbyte-integrations/connectors/destination-e2e-test/Dockerfile b/airbyte-integrations/connectors/destination-e2e-test/Dockerfile index 0b6998be575f..4c0f6c5a50b9 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/Dockerfile +++ b/airbyte-integrations/connectors/destination-e2e-test/Dockerfile @@ -4,7 +4,8 @@ WORKDIR /airbyte ENV APPLICATION destination-e2e-test -ADD build/distributions/${APPLICATION}*.tar /airbyte +COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar +RUN tar xf ${APPLICATION}.tar --strip-components=1 LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/destination-e2e-test diff --git a/airbyte-integrations/connectors/destination-e2e-test/README.md b/airbyte-integrations/connectors/destination-e2e-test/README.md index 43e0d5bc6720..e8f2bee4577a 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/README.md +++ b/airbyte-integrations/connectors/destination-e2e-test/README.md @@ -35,6 +35,9 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-e2e-test:dev disc docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-e2e-test:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` +#### Dev Null Destination +The Dev Null Destination depends on this connector. It only allows the "silent" mode. When this mode is changed, please make sure that the Dev Null Destination is updated and published accordingly as well. + ## Testing We use `JUnit` for Java tests. diff --git a/docs/integrations/destinations/e2e-test.md b/docs/integrations/destinations/e2e-test.md index 064174667af2..d583cdb8fbe9 100644 --- a/docs/integrations/destinations/e2e-test.md +++ b/docs/integrations/destinations/e2e-test.md @@ -14,6 +14,12 @@ This destination is for testing of Airbyte connections. It can be set up as a so ## Mode +### Silent (`/dev/null`) + +**This is the only mode allowed on Airbyte Cloud.** + +This mode works as `/dev/null`. It does nothing about any data from the source connector. This is usually only useful for performance testing of the source connector. + ### Logging This mode logs the data from the source connector. It will log at most 1,000 data entries. @@ -26,10 +32,6 @@ There are the different logging modes to choose from: | Every N-th entry | Log every N-th entry for each data stream. When N=1, it will log every entry. When N=1, it will log every other entry. Etc. | N: the N-th entry to log. Max entry count: max number of entries to log. | | Random sampling | Log a random percentage of the entries for each data stream. | Sampling ratio: a number in range of `[0, 1]`. Optional seed: default to system epoch time. Max entry count: max number of entries to log. | -### `/dev/null` - -This mode works as `/dev/null`. It does nothing about any data from the source connector. This is usually only useful for performance testing of the source connector. - ### Throttling This mode mimics a slow data sync. You can specify the time (in millisecond) of delay between each message from the source is processed. From 8af19b800e8cadcd79b122b620ed15deadb9dd64 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Sat, 18 Dec 2021 15:48:57 -0800 Subject: [PATCH 14/17] Update doc to include changelog for dev null --- docs/integrations/destinations/e2e-test.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/integrations/destinations/e2e-test.md b/docs/integrations/destinations/e2e-test.md index d583cdb8fbe9..44fedf502ee0 100644 --- a/docs/integrations/destinations/e2e-test.md +++ b/docs/integrations/destinations/e2e-test.md @@ -42,7 +42,15 @@ This mode throws an exception after receiving a configurable number of messages. ## CHANGELOG +### E2E Testing Destination + | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--- | | 0.2.0 | 2021-12-16 | [\#8824](https://github.com/airbytehq/airbyte/pull/8824) | Add multiple logging modes. | | 0.1.0 | 2021-05-25 | [\#3290](https://github.com/airbytehq/airbyte/pull/3290) | Create initial version. | + +### Dev Null Destination + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :--- | +| 0.1.0 | 2021-12-16 | [\#8824](https://github.com/airbytehq/airbyte/pull/8824) | Create initial version. | From febd7e1bef2f08374d4d65d866dcecf19bd052fe Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Sat, 18 Dec 2021 16:06:19 -0800 Subject: [PATCH 15/17] Format code --- .../destination/dev_null/DevNullDestination.java | 3 +-- .../destination/dev_null/DevNullDestinationTest.java | 4 ++++ .../destination/e2e_test/logging/BaseLogger.java | 4 ++++ .../destination/e2e_test/logging/EveryNthLogger.java | 4 ++++ .../destination/e2e_test/logging/FirstNLogger.java | 4 ++++ .../e2e_test/logging/LoggingConsumer.java | 7 +++++-- .../e2e_test/logging/RandomSamplingLogger.java | 9 ++++++++- .../destination/e2e_test/logging/TestingLogger.java | 4 ++++ .../e2e_test/logging/TestingLoggerFactory.java | 4 ++++ .../src/main/resources/spec.json | 12 ++++++++++-- .../TestingSilentDestinationAcceptanceTest.java | 4 ++++ 11 files changed, 52 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/destination-dev-null/src/main/java/io/airbyte/integrations/destination/dev_null/DevNullDestination.java b/airbyte-integrations/connectors/destination-dev-null/src/main/java/io/airbyte/integrations/destination/dev_null/DevNullDestination.java index 3e3f0aff6a59..17423cdd6399 100644 --- a/airbyte-integrations/connectors/destination-dev-null/src/main/java/io/airbyte/integrations/destination/dev_null/DevNullDestination.java +++ b/airbyte-integrations/connectors/destination-dev-null/src/main/java/io/airbyte/integrations/destination/dev_null/DevNullDestination.java @@ -33,8 +33,7 @@ public static void main(final String[] args) throws Exception { } /** - * 1. Update the title. - * 2. Only keep the "silent" mode. + * 1. Update the title. 2. Only keep the "silent" mode. */ @Override public ConnectorSpecification modifySpec(final ConnectorSpecification originalSpec) { diff --git a/airbyte-integrations/connectors/destination-dev-null/src/test/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationTest.java b/airbyte-integrations/connectors/destination-dev-null/src/test/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationTest.java index 391fbbaeab9f..eeade0389f25 100644 --- a/airbyte-integrations/connectors/destination-dev-null/src/test/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationTest.java +++ b/airbyte-integrations/connectors/destination-dev-null/src/test/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationTest.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.dev_null; import static org.junit.jupiter.api.Assertions.*; diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/BaseLogger.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/BaseLogger.java index 596369366463..2f6f2c669f0b 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/BaseLogger.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/BaseLogger.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.e2e_test.logging; import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/EveryNthLogger.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/EveryNthLogger.java index 50fd4089551f..53ebadd769c9 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/EveryNthLogger.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/EveryNthLogger.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.e2e_test.logging; import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/FirstNLogger.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/FirstNLogger.java index 10d873f90349..a877b0edce0c 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/FirstNLogger.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/FirstNLogger.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.e2e_test.logging; import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.java index 3ef386f10108..ff7ce7ad282a 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.e2e_test.logging; import io.airbyte.commons.json.Jsons; @@ -67,7 +71,6 @@ public void accept(final AirbyteMessage message) { } @Override - public void close() { - } + public void close() {} } diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/RandomSamplingLogger.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/RandomSamplingLogger.java index b447d012f0c6..93831e0e3c20 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/RandomSamplingLogger.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/RandomSamplingLogger.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.e2e_test.logging; import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; @@ -13,7 +17,10 @@ public class RandomSamplingLogger extends BaseLogger implements TestingLogger { private final double samplingRatio; private final Random random; - public RandomSamplingLogger(final AirbyteStreamNameNamespacePair streamNamePair, final double samplingRatio, final long seed, final int maxEntryCount) { + public RandomSamplingLogger(final AirbyteStreamNameNamespacePair streamNamePair, + final double samplingRatio, + final long seed, + final int maxEntryCount) { super(streamNamePair, maxEntryCount); this.samplingRatio = samplingRatio; this.random = new Random(seed); diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/TestingLogger.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/TestingLogger.java index edc91f1b8a74..b4d2a3f81c78 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/TestingLogger.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/TestingLogger.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.e2e_test.logging; import io.airbyte.protocol.models.AirbyteRecordMessage; diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/TestingLoggerFactory.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/TestingLoggerFactory.java index 5f690b8c6514..4ac900c8584c 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/TestingLoggerFactory.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/TestingLoggerFactory.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.e2e_test.logging; import com.fasterxml.jackson.databind.JsonNode; diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json index bfdcb6a75e7b..7fc0e45a6801 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json @@ -49,7 +49,11 @@ "title": "Every N-th Entry", "description": "For each stream, log every N-th entry with a maximum cap.", "type": "object", - "required": ["logging_type", "nth_entry_to_log", "max_entry_count"], + "required": [ + "logging_type", + "nth_entry_to_log", + "max_entry_count" + ], "properties": { "logging_type": { "type": "string", @@ -79,7 +83,11 @@ "title": "Random Sampling", "description": "For each stream, randomly log a percentage of the entries with a maximum cap.", "type": "object", - "required": ["logging_type", "sampling_ratio", "max_entry_count"], + "required": [ + "logging_type", + "sampling_ratio", + "max_entry_count" + ], "properties": { "logging_type": { "type": "string", diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.java index 19ed30b249e0..b7b7034a5918 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.e2e_test; import static org.junit.jupiter.api.Assertions.assertEquals; From b6c1f8f72ec90835f508c517505a50bd29d96b10 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Sat, 18 Dec 2021 16:14:57 -0800 Subject: [PATCH 16/17] Fix doc --- docs/integrations/destinations/e2e-test.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/destinations/e2e-test.md b/docs/integrations/destinations/e2e-test.md index 44fedf502ee0..3edfaf50155c 100644 --- a/docs/integrations/destinations/e2e-test.md +++ b/docs/integrations/destinations/e2e-test.md @@ -29,7 +29,7 @@ There are the different logging modes to choose from: | Mode | Notes | Parameters | | :--- | :--- | :--- | | First N entries | Log the first N number of data entries for each data stream. | N: how many entries to log. | -| Every N-th entry | Log every N-th entry for each data stream. When N=1, it will log every entry. When N=1, it will log every other entry. Etc. | N: the N-th entry to log. Max entry count: max number of entries to log. | +| Every N-th entry | Log every N-th entry for each data stream. When N=1, it will log every entry. When N=2, it will log every other entry. Etc. | N: the N-th entry to log. Max entry count: max number of entries to log. | | Random sampling | Log a random percentage of the entries for each data stream. | Sampling ratio: a number in range of `[0, 1]`. Optional seed: default to system epoch time. Max entry count: max number of entries to log. | ### Throttling From 3714b988b0f8f870af04305ee519112a36d3f066 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Sun, 19 Dec 2021 00:16:42 -0800 Subject: [PATCH 17/17] Register e2e test destination in seed --- .../init/src/main/resources/icons/airbyte.svg | 3 + .../seed/destination_definitions.yaml | 6 + .../resources/seed/destination_specs.yaml | 159 ++++++++++++++++++ 3 files changed, 168 insertions(+) create mode 100644 airbyte-config/init/src/main/resources/icons/airbyte.svg diff --git a/airbyte-config/init/src/main/resources/icons/airbyte.svg b/airbyte-config/init/src/main/resources/icons/airbyte.svg new file mode 100644 index 000000000000..c19d229cbc90 --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/airbyte.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 3cf381f101c6..2c1ed6155845 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -45,6 +45,12 @@ dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.io/integrations/destinations/dynamodb icon: dynamodb.svg +- name: E2E Testing + destinationDefinitionId: 2eb65e87-983a-4fd7-b3e3-9d9dc6eb8537 + dockerRepository: airbyte/destination-e2e-test + dockerImageTag: 0.2.0 + documentationUrl: https://docs.airbyte.io/integrations/destinations/e2e-test + icon: airbyte.svg - destinationDefinitionId: 68f351a7-2745-4bef-ad7f-996b8e51bb8c name: ElasticSearch dockerRepository: airbyte/destination-elasticsearch diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index fe77429f6aeb..9cc69f41edef 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -769,6 +769,165 @@ supported_destination_sync_modes: - "overwrite" - "append" +- dockerImage: "airbyte/destination-e2e-test:0.2.0" + spec: + documentationUrl: "https://example.com" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "E2E Test Destination Spec" + type: "object" + oneOf: + - title: "Logging" + required: + - "type" + - "logging_config" + properties: + type: + type: "string" + const: "LOGGING" + default: "LOGGING" + logging_config: + title: "Logging Configuration" + type: "object" + description: "Configurate how the messages are logged." + oneOf: + - title: "First N Entries" + description: "Log first N entries per stream." + type: "object" + required: + - "logging_type" + - "max_entry_count" + properties: + logging_type: + type: "string" + enum: + - "FirstN" + default: "FirstN" + max_entry_count: + title: "N" + description: "Number of entries to log. This destination is for\ + \ testing only. So it won't make sense to log infinitely. The\ + \ maximum is 1,000 entries." + type: "number" + default: 100 + examples: + - 100 + minimum: 1 + maximum: 1000 + - title: "Every N-th Entry" + description: "For each stream, log every N-th entry with a maximum cap." + type: "object" + required: + - "logging_type" + - "nth_entry_to_log" + - "max_entry_count" + properties: + logging_type: + type: "string" + enum: + - "EveryNth" + default: "EveryNth" + nth_entry_to_log: + title: "N" + description: "The N-th entry to log for each stream. N starts from\ + \ 1. For example, when N = 1, every entry is logged; when N =\ + \ 2, every other entry is logged; when N = 3, one out of three\ + \ entries is logged." + type: "number" + example: + - 3 + minimum: 1 + maximum: 1000 + max_entry_count: + title: "Max Log Entries" + description: "Max number of entries to log. This destination is\ + \ for testing only. So it won't make sense to log infinitely.\ + \ The maximum is 1,000 entries." + type: "number" + default: 100 + examples: + - 100 + minimum: 1 + maximum: 1000 + - title: "Random Sampling" + description: "For each stream, randomly log a percentage of the entries\ + \ with a maximum cap." + type: "object" + required: + - "logging_type" + - "sampling_ratio" + - "max_entry_count" + properties: + logging_type: + type: "string" + enum: + - "RandomSampling" + default: "RandomSampling" + sampling_ratio: + title: "Sampling Ratio" + description: "A positive floating number smaller than 1." + type: "number" + default: 0.001 + examples: + - 0.001 + minimum: 0 + maximum: 1 + seed: + title: "Random Number Generator Seed" + description: "When the seed is unspecified, the current time millis\ + \ will be used as the seed." + type: "number" + examples: + - 1900 + max_entry_count: + title: "Max Log Entries" + description: "Max number of entries to log. This destination is\ + \ for testing only. So it won't make sense to log infinitely.\ + \ The maximum is 1,000 entries." + type: "number" + default: 100 + examples: + - 100 + minimum: 1 + maximum: 1000 + - title: "Silent" + required: + - "type" + properties: + type: + type: "string" + const: "SILENT" + default: "SILENT" + - title: "Throttled" + required: + - "type" + - "millis_per_record" + properties: + type: + type: "string" + const: "THROTTLED" + default: "THROTTLED" + millis_per_record: + description: "Number of milli-second to pause in between records." + type: "integer" + - title: "Failing" + required: + - "type" + - "num_messages" + properties: + type: + type: "string" + const: "FAILING" + default: "FAILING" + num_messages: + description: "Number of messages after which to fail." + type: "integer" + supportsIncremental: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: + - "overwrite" + - "append" - dockerImage: "airbyte/destination-elasticsearch:0.1.0" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/elasticsearch"