secret coordinate helpers (#6114)

* add some initial tests and todos * add rest of files * format * make json secrets processor availabe as helper functions, not injected classes * get tests working for splitting * save state * complete update * combine working * format * add separate test cases * add oneof test case and fix behavior * combine support for arrays * working string arrays * test object arrays * add unsupported test case * clean up * add airbyte_ prefix * add ability to test individual version bumping * add read only persistence exposure * version handling partially * version handling fully working yay * remove comment typo * add test * throw an exception when the secret can't be found and test for it * test for mutations * remove magic strings * extract coordinate generation into own functions * misc cleanup * massive simplification * mild cleanup * remove json path * fix test run and clean up more * format * misc minor cleanups * remove giant if with filter * significantly clean up split function * significant cleanups and add array of oneof test case * support nestedOneOf test case * add schema validation test and fix failing cases * add javadocs * clarify coordinate conversion * make sense * fix build * revert json secrets processing static-ification to fix build * add in method
airbytehq · Sep 23, 2021 · a8261f4 · a8261f4
1 parent ef6fefc
commit a8261f4
Show file tree

Hide file tree

Showing 64 changed files with 2,262 additions and 23 deletions.
diff --git a/airbyte-config/persistence/build.gradle b/airbyte-config/persistence/build.gradle
@@ -4,7 +4,7 @@ plugins {
 }
 
 dependencies {
-    implementation group: 'commons-io', name: 'commons-io', version: '2.7'
+    implementation 'commons-io:commons-io:2.7'
 
     implementation project(':airbyte-db:lib')
     implementation project(':airbyte-db:jooq')

diff --git a/...rver/converters/JsonSecretsProcessor.java → ...e/split_secrets/JsonSecretsProcessor.java b/...rver/converters/JsonSecretsProcessor.java → ...e/split_secrets/JsonSecretsProcessor.java
@@ -22,7 +22,7 @@
  * SOFTWARE.
  */
 
-package io.airbyte.server.converters;
+package io.airbyte.config.persistence.split_secrets;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ArrayNode;
@@ -38,13 +38,13 @@
 public class JsonSecretsProcessor {
 
   public static String AIRBYTE_SECRET_FIELD = "airbyte_secret";
+  public static final String PROPERTIES_FIELD = "properties";
+
   private static final Logger LOGGER = LoggerFactory.getLogger(JsonSecretsProcessor.class);
 
   @VisibleForTesting
   static String SECRETS_MASK = "**********";
 
-  private static final String PROPERTIES_FIELD = "properties";
-
   /**
    * Returns a copy of the input object wherein any fields annotated with "airbyte_secret" in the
    * input schema are masked.
@@ -55,6 +55,8 @@ public class JsonSecretsProcessor {
    * @param schema Schema containing secret annotations
    * @param obj Object containing potentially secret fields
    */
+  // todo: fix bug where this doesn't handle non-oneof nesting or just arrays
+  // see: https://github.com/airbytehq/airbyte/issues/6393
   public JsonNode maskSecrets(JsonNode obj, JsonNode schema) {
     // if schema is an object and has a properties field
     if (!canBeProcessed(schema)) {
@@ -90,7 +92,7 @@ public JsonNode maskSecrets(JsonNode obj, JsonNode schema) {
     return copy;
   }
 
-  private static Optional<String> findJsonCombinationNode(JsonNode node) {
+  public static Optional<String> findJsonCombinationNode(JsonNode node) {
     for (String combinationNode : List.of("allOf", "anyOf", "oneOf")) {
       if (node.has(combinationNode) && node.get(combinationNode).isArray()) {
         return Optional.of(combinationNode);
@@ -147,11 +149,11 @@ public JsonNode copySecrets(JsonNode src, JsonNode dst, JsonNode schema) {
     return dstCopy;
   }
 
-  private static boolean isSecret(JsonNode obj) {
+  public static boolean isSecret(JsonNode obj) {
     return obj.isObject() && obj.has(AIRBYTE_SECRET_FIELD) && obj.get(AIRBYTE_SECRET_FIELD).asBoolean();
   }
 
-  private static boolean canBeProcessed(JsonNode schema) {
+  public static boolean canBeProcessed(JsonNode schema) {
     return schema.isObject() && schema.has(PROPERTIES_FIELD) && schema.get(PROPERTIES_FIELD).isObject();
   }
 

diff --git a/...ce/src/main/java/io/airbyte/config/persistence/split_secrets/MemorySecretPersistence.java b/...ce/src/main/java/io/airbyte/config/persistence/split_secrets/MemorySecretPersistence.java
@@ -0,0 +1,48 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Airbyte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package io.airbyte.config.persistence.split_secrets;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * Map-based implementation of a {@link SecretPersistence} used for unit testing.
+ */
+public class MemorySecretPersistence implements SecretPersistence {
+
+  final Map<SecretCoordinate, String> secretMap = new HashMap<>();
+
+  @Override
+  public Optional<String> read(SecretCoordinate coordinate) {
+    return Optional.ofNullable(secretMap.get(coordinate));
+  }
+
+  @Override
+  public void write(SecretCoordinate coordinate, String payload) {
+    secretMap.put(coordinate, payload);
+  }
+
+}
diff --git a/.../src/main/java/io/airbyte/config/persistence/split_secrets/ReadOnlySecretPersistence.java b/.../src/main/java/io/airbyte/config/persistence/split_secrets/ReadOnlySecretPersistence.java
@@ -0,0 +1,38 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Airbyte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package io.airbyte.config.persistence.split_secrets;
+
+import java.util.Optional;
+
+/**
+ * Provides a read-only interface to a backing secrets store similar to {@link SecretPersistence}.
+ * In practice, the functionality should be provided by a {@link SecretPersistence#read function.
+ */
+@FunctionalInterface
+public interface ReadOnlySecretPersistence {
+
+  Optional<String> read(SecretCoordinate coordinate);
+
+}
diff --git a/...rsistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretCoordinate.java b/...rsistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretCoordinate.java
@@ -0,0 +1,106 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Airbyte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package io.airbyte.config.persistence.split_secrets;
+
+import com.google.api.client.util.Preconditions;
+import java.util.Objects;
+
+/**
+ * A secret coordinate represents a specific secret at a specific version stored within a
+ * {@link SecretPersistence}.
+ *
+ * We use "coordinate base" to refer to a string reference to a secret without versioning
+ * information. We use "full coordinate" to refer to a string reference that includes both the
+ * coordinate base and version-specific information. You should be able to go from a "full
+ * coordinate" to a coordinate object and back without loss of information.
+ *
+ * Example coordinate base:
+ * airbyte_workspace_e0eb0554-ffe0-4e9c-9dc0-ed7f52023eb2_secret_9eba44d8-51e7-48f1-bde2-619af0e42c22
+ *
+ * Example full coordinate:
+ * airbyte_workspace_e0eb0554-ffe0-4e9c-9dc0-ed7f52023eb2_secret_9eba44d8-51e7-48f1-bde2-619af0e42c22_v1
+ *
+ * This coordinate system was designed to work well with Google Secrets Manager but should work with
+ * other secret storage backends as well.
+ */
+public class SecretCoordinate {
+
+  private final String coordinateBase;
+  private final long version;
+
+  public SecretCoordinate(final String coordinateBase, final long version) {
+    this.coordinateBase = coordinateBase;
+    this.version = version;
+  }
+
+  /**
+   * Used to turn a full string coordinate into a coordinate object using a full coordinate generated
+   * by {@link SecretsHelpers#getCoordinate}.
+   *
+   * This will likely need refactoring if we end up using a secret store that doesn't allow the same
+   * format of full coordinate.
+   *
+   * @param fullCoordinate coordinate with version
+   * @return secret coordinate object
+   */
+  public static SecretCoordinate fromFullCoordinate(String fullCoordinate) {
+    final var splits = fullCoordinate.split("_v");
+    Preconditions.checkArgument(splits.length == 2);
+
+    return new SecretCoordinate(splits[0], Long.parseLong(splits[1]));
+  }
+
+  public String getCoordinateBase() {
+    return coordinateBase;
+  }
+
+  public long getVersion() {
+    return version;
+  }
+
+  public String getFullCoordinate() {
+    return coordinateBase + "_v" + version;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o)
+      return true;
+    if (o == null || getClass() != o.getClass())
+      return false;
+    SecretCoordinate that = (SecretCoordinate) o;
+    return toString().equals(that.toString());
+  }
+
+  /**
+   * The hash code is computed using the {@link SecretCoordinate#getFullCoordinate} because the full
+   * secret coordinate should be a valid unique representation of the secret coordinate.
+   */
+  @Override
+  public int hashCode() {
+    return Objects.hash(getFullCoordinate());
+  }
+
+}
diff --git a/...sistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretPersistence.java b/...sistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretPersistence.java
@@ -0,0 +1,40 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Airbyte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package io.airbyte.config.persistence.split_secrets;
+
+import java.util.Optional;
+
+/**
+ * Provides the ability to read and write secrets to a backing store. Assumes that secret payloads
+ * are always strings. See {@link SecretCoordinate} for more information on how secrets are
+ * identified.
+ */
+public interface SecretPersistence {
+
+  Optional<String> read(SecretCoordinate coordinate);
+
+  void write(SecretCoordinate coordinate, String payload);
+
+}