🎉 Testing destination: multiple logging modes

airbyte-config/init/src/main/resources/seed/destination_definitions.yaml

@@ -45,6 +45,12 @@
   dockerImageTag: 0.1.0
   documentationUrl: https://docs.airbyte.io/integrations/destinations/dynamodb
   icon: dynamodb.svg
+- name: E2E Testing
+  destinationDefinitionId: 2eb65e87-983a-4fd7-b3e3-9d9dc6eb8537
+  dockerRepository: airbyte/destination-e2e-test
+  dockerImageTag: 0.2.0
+  documentationUrl: https://docs.airbyte.io/integrations/destinations/e2e-test
+  icon: airbyte.svg
 - destinationDefinitionId: 68f351a7-2745-4bef-ad7f-996b8e51bb8c
   name: ElasticSearch
   dockerRepository: airbyte/destination-elasticsearch

airbyte-config/init/src/main/resources/seed/destination_specs.yaml

@@ -769,6 +769,165 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
+- dockerImage: "airbyte/destination-e2e-test:0.2.0"
+  spec:
+    documentationUrl: "https://example.com"
+    connectionSpecification:
+      $schema: "http://json-schema.org/draft-07/schema#"
+      title: "E2E Test Destination Spec"
+      type: "object"
+      oneOf:
+      - title: "Logging"
+        required:
+        - "type"
+        - "logging_config"
+        properties:
+          type:
+            type: "string"
+            const: "LOGGING"
+            default: "LOGGING"
+          logging_config:
+            title: "Logging Configuration"
+            type: "object"
+            description: "Configurate how the messages are logged."
+            oneOf:
+            - title: "First N Entries"
+              description: "Log first N entries per stream."
+              type: "object"
+              required:
+              - "logging_type"
+              - "max_entry_count"
+              properties:
+                logging_type:
+                  type: "string"
+                  enum:
+                  - "FirstN"
+                  default: "FirstN"
+                max_entry_count:
+                  title: "N"
+                  description: "Number of entries to log. This destination is for\
+                    \ testing only. So it won't make sense to log infinitely. The\
+                    \ maximum is 1,000 entries."
+                  type: "number"
+                  default: 100
+                  examples:
+                  - 100
+                  minimum: 1
+                  maximum: 1000
+            - title: "Every N-th Entry"
+              description: "For each stream, log every N-th entry with a maximum cap."
+              type: "object"
+              required:
+              - "logging_type"
+              - "nth_entry_to_log"
+              - "max_entry_count"
+              properties:
+                logging_type:
+                  type: "string"
+                  enum:
+                  - "EveryNth"
+                  default: "EveryNth"
+                nth_entry_to_log:
+                  title: "N"
+                  description: "The N-th entry to log for each stream. N starts from\
+                    \ 1. For example, when N = 1, every entry is logged; when N =\
+                    \ 2, every other entry is logged; when N = 3, one out of three\
+                    \ entries is logged."
+                  type: "number"
+                  example:
+                  - 3
+                  minimum: 1
+                  maximum: 1000
+                max_entry_count:
+                  title: "Max Log Entries"
+                  description: "Max number of entries to log. This destination is\
+                    \ for testing only. So it won't make sense to log infinitely.\
+                    \ The maximum is 1,000 entries."
+                  type: "number"
+                  default: 100
+                  examples:
+                  - 100
+                  minimum: 1
+                  maximum: 1000
+            - title: "Random Sampling"
+              description: "For each stream, randomly log a percentage of the entries\
+                \ with a maximum cap."
+              type: "object"
+              required:
+              - "logging_type"
+              - "sampling_ratio"
+              - "max_entry_count"
+              properties:
+                logging_type:
+                  type: "string"
+                  enum:
+                  - "RandomSampling"
+                  default: "RandomSampling"
+                sampling_ratio:
+                  title: "Sampling Ratio"
+                  description: "A positive floating number smaller than 1."
+                  type: "number"
+                  default: 0.001
+                  examples:
+                  - 0.001
+                  minimum: 0
+                  maximum: 1
+                seed:
+                  title: "Random Number Generator Seed"
+                  description: "When the seed is unspecified, the current time millis\
+                    \ will be used as the seed."
+                  type: "number"
+                  examples:
+                  - 1900
+                max_entry_count:
+                  title: "Max Log Entries"
+                  description: "Max number of entries to log. This destination is\
+                    \ for testing only. So it won't make sense to log infinitely.\
+                    \ The maximum is 1,000 entries."
+                  type: "number"
+                  default: 100
+                  examples:
+                  - 100
+                  minimum: 1
+                  maximum: 1000
+      - title: "Silent"
+        required:
+        - "type"
+        properties:
+          type:
+            type: "string"
+            const: "SILENT"
+            default: "SILENT"
+      - title: "Throttled"
+        required:
+        - "type"
+        - "millis_per_record"
+        properties:
+          type:
+            type: "string"
+            const: "THROTTLED"
+            default: "THROTTLED"
+          millis_per_record:
+            description: "Number of milli-second to pause in between records."
+            type: "integer"
+      - title: "Failing"
+        required:
+        - "type"
+        - "num_messages"
+        properties:
+          type:
+            type: "string"
+            const: "FAILING"
+            default: "FAILING"
+          num_messages:
+            description: "Number of messages after which to fail."
+            type: "integer"
+    supportsIncremental: true
+    supportsNormalization: false
+    supportsDBT: false
+    supported_destination_sync_modes:
+    - "overwrite"
+    - "append"
 - dockerImage: "airbyte/destination-elasticsearch:0.1.0"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/elasticsearch"

airbyte-integrations/builds.md

@@ -112,7 +112,9 @@
 | ClickHouse   | [![destination-clickhouse](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-clickhouse%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-clickhouse) |
 | Cassandra   | [![destination-cassandra](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-cassandra%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-cassandra) |
 | Databricks | (Temporarily Not Available) |
+| Dev Null   | [![destination-dev-null](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-dev-null%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-dev-null) |
 | Elasticsearch | (Temporarily Not Available) |
+| End-to-End Testing   | [![destination-e2e-test](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-e2e-test%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-e2e-test) |
 | Google Cloud Storage (GCS) | [![destination-gcs](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-gcs%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-gcs) |
 | Google Firestore | [![destination-firestore](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-firestore%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-firestore) |
 | Google PubSub   | [![destination-pubsub](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-pubsub%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-pubsub) |

airbyte-integrations/connectors/destination-dev-null/.dockerignore

@@ -0,0 +1,3 @@
+*
+!Dockerfile
+!build

airbyte-integrations/connectors/destination-dev-null/Dockerfile

@@ -0,0 +1,10 @@
+FROM airbyte/integration-base-java:dev
+
+WORKDIR /airbyte
+ENV APPLICATION destination-dev-null
+
+COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
+RUN tar xf ${APPLICATION}.tar --strip-components=1
+
+LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.name=airbyte/destination-dev-null

airbyte-integrations/connectors/destination-dev-null/README.md

@@ -0,0 +1,51 @@
+# Destination Dev Null
+
+This destination is a "safe" version of the [E2E Test destination](https://docs.airbyte.io/integrations/destinations/e2e-test). It only allows the "silent" mode. 
+
+## Local development
+
+#### Building via Gradle
+From the Airbyte repository root, run:
+```
+./gradlew :airbyte-integrations:connectors:destination-dev-null:build
+```
+
+### Locally running the connector docker image
+
+#### Build
+Build the connector image via Gradle:
+```
+./gradlew :airbyte-integrations:connectors:destination-dev-null:airbyteDocker
+```
+When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in
+the Dockerfile.
+
+#### Run
+Then run any of the connector commands as follows:
+```
+docker run --rm airbyte/destination-dev-null:dev spec
+docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-dev-null:dev check --config /secrets/config.json
+docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-dev-null:dev discover --config /secrets/config.json
+docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-dev-null:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
+```
+
+### Using gradle to run tests
+All commands should be run from airbyte project root.
+To run unit tests:
+```
+./gradlew :airbyte-integrations:connectors:destination-dev-null:unitTest
+```
+To run acceptance and custom integration tests:
+```
+./gradlew :airbyte-integrations:connectors:destination-dev-null:integrationTest
+```
+
+## Dependency Management
+
+### Publishing a new version of the connector
+You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
+1. Make sure your changes are passing unit and integration tests.
+1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)).
+1. Create a Pull Request.
+1. Pat yourself on the back for being an awesome contributor.
+1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.

airbyte-integrations/connectors/destination-dev-null/build.gradle

@@ -0,0 +1,20 @@
+plugins {
+    id 'application'
+    id 'airbyte-docker'
+    id 'airbyte-integration-test-java'
+}
+
+application {
+    mainClass = 'io.airbyte.integrations.destination.dev_null.DevNullDestination'
+}
+
+dependencies {
+    implementation project(':airbyte-config:models')
+    implementation project(':airbyte-protocol:models')
+    implementation project(':airbyte-integrations:bases:base-java')
+    implementation project(':airbyte-integrations:connectors:destination-e2e-test')
+    implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
+
+    integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test')
+    integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-dev-null')
+}

airbyte-integrations/connectors/destination-dev-null/src/main/java/io/airbyte/integrations/destination/dev_null/DevNullDestination.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.integrations.destination.dev_null;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.airbyte.commons.json.Jsons;
+import io.airbyte.integrations.base.Destination;
+import io.airbyte.integrations.base.IntegrationRunner;
+import io.airbyte.integrations.base.spec_modification.SpecModifyingDestination;
+import io.airbyte.integrations.destination.e2e_test.TestingDestinations;
+import io.airbyte.protocol.models.ConnectorSpecification;
+import java.util.Iterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DevNullDestination extends SpecModifyingDestination implements Destination {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(DevNullDestination.class);
+  private static final String DEV_NULL_DESTINATION_TITLE = "Dev Null Destination Spec";
+
+  public DevNullDestination() {
+    super(new TestingDestinations());
+  }
+
+  public static void main(final String[] args) throws Exception {
+    LOGGER.info("Starting destination: {}", DevNullDestination.class);
+    new IntegrationRunner(new DevNullDestination()).run(args);
+    LOGGER.info("Completed destination: {}", DevNullDestination.class);
+  }
+
+  /**
+   * 1. Update the title. 2. Only keep the "silent" mode.
+   */
+  @Override
+  public ConnectorSpecification modifySpec(final ConnectorSpecification originalSpec) {
+    final ConnectorSpecification spec = Jsons.clone(originalSpec);
+
+    ((ObjectNode) spec.getConnectionSpecification()).put("title", DEV_NULL_DESTINATION_TITLE);
+
+    final ArrayNode types = (ArrayNode) spec.getConnectionSpecification().get("oneOf");
+    final Iterator<JsonNode> typesIterator = types.elements();
+    while (typesIterator.hasNext()) {
+      final JsonNode typeNode = typesIterator.next();
+      if (!typeNode.get("properties").get("type").get("const").asText().equalsIgnoreCase("silent")) {
+        typesIterator.remove();
+      }
+    }
+    return spec;
+  }
+
+}

airbyte-integrations/connectors/destination-dev-null/src/test-integration/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.integrations.destination.dev_null;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import io.airbyte.commons.json.Jsons;
+import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
+import io.airbyte.protocol.models.AirbyteMessage;
+import io.airbyte.protocol.models.AirbyteRecordMessage;
+import java.util.Collections;
+import java.util.List;
+
+public class DevNullDestinationAcceptanceTest extends DestinationAcceptanceTest {
+
+  @Override
+  protected String getImageName() {
+    return "airbyte/destination-dev-null:dev";
+  }
+
+  @Override
+  protected JsonNode getConfig() {
+    return Jsons.jsonNode(Collections.singletonMap("type", "SILENT"));
+  }
+
+  @Override
+  protected JsonNode getFailCheckConfig() {
+    return Jsons.jsonNode(Collections.singletonMap("type", "invalid"));
+  }
+
+  @Override
+  protected List<JsonNode> retrieveRecords(final TestDestinationEnv testEnv,
+                                           final String streamName,
+                                           final String namespace,
+                                           final JsonNode streamSchema) {
+    return Collections.emptyList();
+  }
+
+  @Override
+  protected void setup(final TestDestinationEnv testEnv) {
+    // do nothing
+  }
+
+  @Override
+  protected void tearDown(final TestDestinationEnv testEnv) {
+    // do nothing
+  }
+
+  @Override
+  protected void assertSameMessages(final List<AirbyteMessage> expected,
+                                    final List<AirbyteRecordMessage> actual,
+                                    final boolean pruneAirbyteInternalFields) {
+    assertEquals(0, actual.size());
+  }
+
+}