🎉 Integration Testing for SSH using a docker container | Postgres Sou…

…rce and Destination update integration tests using ssh bastion in docker container (#6312)

* ssh-test

* add authentification via ssh tunnel with bastion docker host and postgres testcontainer

* created SshBastion class in base-java module

* implement Postgres source basic ssh tunneling connection for integration tests

* implement Postgres source ssh tunneling connection and refactoring SshBastion

* generate keys inside a bastion container

* remove throwing Exception from startTestContainers method

* fix checkstyle

* add documentation and changelog for Posthres source and destination

* update documentation for ssh readme.md | update version fo Postgres source and destination to 0.3.12

* update version of Postgres source and destination to 0.3.12

* removed static variables, removed version bump, rename class to SshBastionContainer, removed ci credentials for ssh Postgres Source and Destination

Co-authored-by: vmaltsev <vitalii.maltsev@globallogic.com>

airbyte-integrations/bases/base-java/build.gradle

@@ -14,6 +14,8 @@ dependencies {
 
     implementation project(':airbyte-protocol:models')
     implementation project(":airbyte-json-validation")
+    implementation "org.testcontainers:testcontainers:1.15.1"
+    implementation "org.testcontainers:jdbc:1.15.1"
 
     implementation files(project(':airbyte-integrations:bases:base').airbyteDocker.outputs)
 }

airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshBastionContainer.java

@@ -0,0 +1,101 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Airbyte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package io.airbyte.integrations.base.ssh;
+
+import static io.airbyte.integrations.base.ssh.SshTunnel.TunnelMethod.SSH_KEY_AUTH;
+import static io.airbyte.integrations.base.ssh.SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.collect.ImmutableMap;
+import io.airbyte.commons.json.Jsons;
+import java.io.IOException;
+import java.util.Objects;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.JdbcDatabaseContainer;
+import org.testcontainers.containers.Network;
+import org.testcontainers.images.builder.ImageFromDockerfile;
+
+public class SshBastionContainer {
+
+  private static final String SSH_USER = "sshuser";
+  private static final String SSH_PASSWORD = "secret";
+  private Network network;
+  private GenericContainer bastion;
+
+  public void initAndStartBastion() {
+    network = Network.newNetwork();
+    bastion = new GenericContainer(
+        new ImageFromDockerfile("bastion-test")
+            .withFileFromClasspath("Dockerfile", "bastion/Dockerfile"))
+                .withNetwork(network)
+                .withExposedPorts(22);
+    bastion.start();
+  }
+
+  public JsonNode getTunnelConfig(SshTunnel.TunnelMethod tunnelMethod, ImmutableMap.Builder<Object, Object> builderWithSchema)
+      throws IOException, InterruptedException {
+
+    return Jsons.jsonNode(builderWithSchema
+        .put("tunnel_method", Jsons.jsonNode(ImmutableMap.builder()
+            .put("tunnel_host",
+                Objects.requireNonNull(bastion.getContainerInfo().getNetworkSettings()
+                    .getNetworks()
+                    .get(((Network.NetworkImpl) network).getName())
+                    .getIpAddress()))
+            .put("tunnel_method", tunnelMethod)
+            .put("tunnel_port", bastion.getExposedPorts().get(0))
+            .put("tunnel_user", SSH_USER)
+            .put("tunnel_user_password", tunnelMethod.equals(SSH_PASSWORD_AUTH) ? SSH_PASSWORD : "")
+            .put("ssh_key", tunnelMethod.equals(SSH_KEY_AUTH) ? bastion.execInContainer("cat", "var/bastion/id_rsa").getStdout() : "")
+            .build()))
+        .build());
+  }
+
+  public ImmutableMap.Builder<Object, Object> getBasicDbConfigBuider(JdbcDatabaseContainer<?> db) {
+    return ImmutableMap.builder()
+        .put("host", Objects.requireNonNull(db.getContainerInfo().getNetworkSettings()
+            .getNetworks()
+            .get(((Network.NetworkImpl) getNetWork()).getName())
+            .getIpAddress()))
+        .put("username", db.getUsername())
+        .put("password", db.getPassword())
+        .put("port", db.getExposedPorts().get(0))
+        .put("database", db.getDatabaseName())
+        .put("ssl", false);
+  }
+
+  public Network getNetWork() {
+    return this.network;
+  }
+
+  public void stopAndCloseContainers(JdbcDatabaseContainer<?> db) {
+    db.stop();
+    db.close();
+    bastion.stop();
+    bastion.close();
+    network.close();
+  }
+
+}

airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/readme.md

@@ -50,6 +50,13 @@ There is a tradeoff here.
 * (Good) The connector code does not need to know anything about SSH, it can just operate on the host and port it gets (and we let SSH Tunnel handle swapping the names for us) which makes writing a connector easier.
 * (Bad) The downside is that the `SshTunnel` logic is more complicated because it is absorbing all of this name swapping so that neither user nor connector developer need to worry about it. In our estimation, the good outweighs the extra complexity incurred here.
 
+
+### Acceptance Testing via ssh tunnel using SshBastion and JdbcDatabaseContainer in Docker 
+1. The `SshBastion` class provides 3 helper functions:
+   `initAndStartBastion()`to initialize and start SSH Bastion server in Docker test container and creates new `Network` for bastion and tested jdbc container
+   `getTunnelConfig()`which return JsoneNode with all necessary configuration to establish ssh tunnel. Connection configuration for integration tests is now taken directly from container settings and does not require a real database connection
+   `stopAndCloseContainers` to stop and close SshBastion and JdbcDatabaseContainer at the end of the test
+
 ## Future Work
 * Add unit / integration testing for `ssh` package.
 * Restructure spec so that instead of having `SSH Key Authentication` or `Password Authentication` options for `tunnel_method`, just have an `SSH` option and then within that `SSH` option have a `oneOf` for password or key. This is blocked because we cannot use `oneOf`s nested in `oneOf`s.

airbyte-integrations/bases/base-java/src/main/resources/bastion/Dockerfile

@@ -0,0 +1,25 @@
+FROM ubuntu:18.04
+
+RUN apt-get update && apt-get install -y openssh-server
+RUN apt-get install -y apt-utils
+RUN mkdir /var/run/sshd
+RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
+
+RUN useradd -m -s /bin/bash sshuser
+RUN echo "sshuser:secret" | chpasswd
+
+RUN mkdir /var/bastion
+RUN ssh-keygen -m PEM -t rsa -b 4096 -C "test-container-bastion" -P "" -f /var/bastion/id_rsa -q
+RUN install -D /var/bastion/id_rsa.pub /home/sshuser/.ssh/authorized_keys
+
+RUN chown -R sshuser:sshuser /home/sshuser/.ssh
+RUN chmod 600 /home/sshuser/.ssh/authorized_keys
+
+RUN mkdir /root/.ssh
+
+RUN apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+EXPOSE 22
+
+CMD    ["/usr/sbin/sshd", "-D"]

airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshKeyPostgresDestinationAcceptanceTest.java

@@ -24,13 +24,13 @@
 
 package io.airbyte.integrations.destination.postgres;
 
-import java.nio.file.Path;
+import io.airbyte.integrations.base.ssh.SshTunnel;
 
 public class SshKeyPostgresDestinationAcceptanceTest extends SshPostgresDestinationAcceptanceTest {
 
   @Override
-  public Path getConfigFilePath() {
-    return Path.of("secrets/ssh-key-config.json");
+  public SshTunnel.TunnelMethod getTunnelMethod() {
+    return SshTunnel.TunnelMethod.SSH_KEY_AUTH;
   }
 
 }

airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPasswordPostgresDestinationAcceptanceTest.java

@@ -24,13 +24,13 @@
 
 package io.airbyte.integrations.destination.postgres;
 
-import java.nio.file.Path;
+import io.airbyte.integrations.base.ssh.SshTunnel;
 
 public class SshPasswordPostgresDestinationAcceptanceTest extends SshPostgresDestinationAcceptanceTest {
 
   @Override
-  public Path getConfigFilePath() {
-    return Path.of("secrets/ssh-pwd-config.json");
+  public SshTunnel.TunnelMethod getTunnelMethod() {
+    return SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH;
   }
 
 }

airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPostgresDestinationAcceptanceTest.java

@@ -27,24 +27,25 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import io.airbyte.commons.functional.CheckedFunction;
-import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.json.Jsons;
 import io.airbyte.db.Database;
 import io.airbyte.db.Databases;
 import io.airbyte.integrations.base.JavaBaseConstants;
+import io.airbyte.integrations.base.ssh.SshBastionContainer;
 import io.airbyte.integrations.base.ssh.SshTunnel;
 import io.airbyte.integrations.destination.ExtendedNameTransformer;
 import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
-import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.jooq.JSONFormat;
 import org.jooq.JSONFormat.RecordFormat;
+import org.testcontainers.containers.PostgreSQLContainer;
 
 // todo (cgardens) - likely some of this could be further de-duplicated with
 // PostgresDestinationAcceptanceTest.
+
 /**
  * Abstract class that allows us to avoid duplicating testing logic for testing SSH with a key file
  * or with a password.
@@ -54,30 +55,24 @@ public abstract class SshPostgresDestinationAcceptanceTest extends DestinationAc
   private static final JSONFormat JSON_FORMAT = new JSONFormat().recordFormat(RecordFormat.OBJECT);
 
   private final ExtendedNameTransformer namingResolver = new ExtendedNameTransformer();
+  private static final String schemaName = RandomStringUtils.randomAlphabetic(8).toLowerCase();
+  private static PostgreSQLContainer<?> db;
+  private final SshBastionContainer bastion = new SshBastionContainer();
 
-  private String schemaName;
-
-  public abstract Path getConfigFilePath();
+  public abstract SshTunnel.TunnelMethod getTunnelMethod();
 
   @Override
   protected String getImageName() {
     return "airbyte/destination-postgres:dev";
   }
 
   @Override
-  protected JsonNode getConfig() {
-    final JsonNode config = getConfigFromSecretsFile();
-    // do everything in a randomly generated schema so that we can wipe it out at the end.
-    ((ObjectNode) config).put("schema", schemaName);
-    return config;
-  }
-
-  private JsonNode getConfigFromSecretsFile() {
-    return Jsons.deserialize(IOs.readFile(getConfigFilePath()));
+  protected JsonNode getConfig() throws Exception {
+    return bastion.getTunnelConfig(getTunnelMethod(), bastion.getBasicDbConfigBuider(db).put("schema", schemaName));
   }
 
   @Override
-  protected JsonNode getFailCheckConfig() {
+  protected JsonNode getFailCheckConfig() throws Exception {
     final JsonNode clone = Jsons.clone(getConfig());
     ((ObjectNode) clone).put("password", "wrong password");
     return clone;
@@ -162,8 +157,9 @@ private List<JsonNode> retrieveRecordsFromTable(final String tableName, final St
 
   @Override
   protected void setup(final TestDestinationEnv testEnv) throws Exception {
+
+    startTestContainers();
     // do everything in a randomly generated schema so that we can wipe it out at the end.
-    schemaName = RandomStringUtils.randomAlphabetic(8).toLowerCase();
     SshTunnel.sshWrap(
         getConfig(),
         PostgresDestination.HOST_KEY,
@@ -173,6 +169,17 @@ protected void setup(final TestDestinationEnv testEnv) throws Exception {
         });
   }
 
+  private void startTestContainers() {
+    bastion.initAndStartBastion();
+    initAndStartJdbcContainer();
+  }
+
+  private void initAndStartJdbcContainer() {
+    db = new PostgreSQLContainer<>("postgres:13-alpine")
+        .withNetwork(bastion.getNetWork());
+    db.start();
+  }
+
   @Override
   protected void tearDown(final TestDestinationEnv testEnv) throws Exception {
     // blow away the test schema at the end.
@@ -183,6 +190,8 @@ protected void tearDown(final TestDestinationEnv testEnv) throws Exception {
         mangledConfig -> {
           getDatabaseFromConfig(mangledConfig).query(ctx -> ctx.fetch(String.format("DROP SCHEMA %s CASCADE;", schemaName)));
         });
+
+    bastion.stopAndCloseContainers(db);
   }
 
 }

airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java

@@ -26,9 +26,12 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.collect.Lists;
-import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.json.Jsons;
+import io.airbyte.db.Database;
+import io.airbyte.db.Databases;
+import io.airbyte.integrations.base.ssh.SshBastionContainer;
 import io.airbyte.integrations.base.ssh.SshHelpers;
+import io.airbyte.integrations.base.ssh.SshTunnel;
 import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest;
 import io.airbyte.integrations.standardtest.source.TestDestinationEnv;
 import io.airbyte.protocol.models.CatalogHelpers;
@@ -39,30 +42,67 @@
 import io.airbyte.protocol.models.Field;
 import io.airbyte.protocol.models.JsonSchemaPrimitive;
 import io.airbyte.protocol.models.SyncMode;
-import java.nio.file.Path;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import org.jooq.SQLDialect;
+import org.testcontainers.containers.PostgreSQLContainer;
 
 public abstract class AbstractSshPostgresSourceAcceptanceTest extends SourceAcceptanceTest {
 
   private static final String STREAM_NAME = "public.id_and_name";
   private static final String STREAM_NAME2 = "public.starships";
+  private PostgreSQLContainer<?> db;
+  private final SshBastionContainer bastion = new SshBastionContainer();
+  private static JsonNode config;
 
-  private JsonNode config;
-
-  public abstract Path getConfigFilePath();
+  public abstract SshTunnel.TunnelMethod getTunnelMethod();
 
   // todo (cgardens) - dynamically create data by generating a database with a random name instead of
   // requiring data to already be in place.
   @Override
   protected void setupEnvironment(final TestDestinationEnv environment) throws Exception {
-    config = Jsons.deserialize(IOs.readFile(getConfigFilePath()));
+    startTestContainers();
+    config = bastion.getTunnelConfig(getTunnelMethod(), bastion.getBasicDbConfigBuider(db));
+    populateDatabaseTestData();
+
+  }
+
+  private void startTestContainers() {
+    bastion.initAndStartBastion();
+    initAndStartJdbcContainer();
+  }
+
+  private void initAndStartJdbcContainer() {
+    db = new PostgreSQLContainer<>("postgres:13-alpine").withNetwork(bastion.getNetWork());
+    db.start();
+  }
+
+  private static void populateDatabaseTestData() throws Exception {
+    final Database database = Databases.createDatabase(
+        config.get("username").asText(),
+        config.get("password").asText(),
+        String.format("jdbc:postgresql://%s:%s/%s",
+            config.get("host").asText(),
+            config.get("port").asText(),
+            config.get("database").asText()),
+        "org.postgresql.Driver",
+        SQLDialect.POSTGRES);
+
+    database.query(ctx -> {
+      ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));");
+      ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'),  (2, 'crusher'), (3, 'vash');");
+      ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));");
+      ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'),  (2, 'defiant'), (3, 'yamato');");
+      return null;
+    });
+
+    database.close();
   }
 
   @Override
   protected void tearDown(final TestDestinationEnv testEnv) {
-
+    bastion.stopAndCloseContainers(db);
   }
 
   @Override

airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshKeyPostgresSourceAcceptanceTest.java

@@ -24,13 +24,13 @@
 
 package io.airbyte.integrations.io.airbyte.integration_tests.sources;
 
-import java.nio.file.Path;
+import io.airbyte.integrations.base.ssh.SshTunnel;
 
 public class SshKeyPostgresSourceAcceptanceTest extends AbstractSshPostgresSourceAcceptanceTest {
 
   @Override
-  public Path getConfigFilePath() {
-    return Path.of("secrets/ssh-key-config.json");
+  public SshTunnel.TunnelMethod getTunnelMethod() {
+    return SshTunnel.TunnelMethod.SSH_KEY_AUTH;
   }
 
 }