ci-credentials: update GSM secrets with updated configuration values

.github/workflows/publish-command.yml

@@ -283,12 +283,12 @@ jobs:
       - name: Write Integration Test Credentials for ${{ matrix.connector }}
         run: |
           source venv/bin/activate
-          ci_credentials ${{ matrix.connector }}
+          ci_credentials ${{ matrix.connector }} write-to-storage
           # normalization also runs destination-specific tests, so fetch their creds also
           if [ 'bases/base-normalization' = "${{ matrix.connector }}" ] || [ 'base-normalization' = "${{ matrix.connector }}" ]; then
-            ci_credentials destination-bigquery
-            ci_credentials destination-postgres
-            ci_credentials destination-snowflake
+            ci_credentials destination-bigquery write-to-storage
+            ci_credentials destination-postgres write-to-storage
+            ci_credentials destination-snowflake write-to-storage
           fi
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
@@ -317,9 +317,21 @@ jobs:
         with:
           command: |
             echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD}
-            ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache   
+            ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache
           attempt_limit: 3
           attempt_delay: 5000 in # ms
+      - name: Update Integration Test Credentials after test run for ${{ github.event.inputs.connector }}
+        run: |
+          source venv/bin/activate
+          ci_credentials ${{ matrix.connector }} update-secrets
+          # normalization also runs destination-specific tests, so fetch their creds also
+          if [ 'bases/base-normalization' = "${{ matrix.connector }}" ] || [ 'base-normalization' = "${{ matrix.connector }}" ]; then
+            ci_credentials destination-bigquery update-secrets
+            ci_credentials destination-postgres update-secrets
+            ci_credentials destination-snowflake update-secrets
+          fi
+        env:
+          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
       - name: Create Sentry Release
         if: startsWith(matrix.connector, 'connectors') && success()
         run: |

.github/workflows/test-command.yml

@@ -110,12 +110,12 @@ jobs:
       - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }}
         run: |
           source venv/bin/activate
-          ci_credentials ${{ github.event.inputs.connector }}
+          ci_credentials ${{ github.event.inputs.connector }} write-to-storage
           # normalization also runs destination-specific tests, so fetch their creds also
           if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then
-            ci_credentials destination-bigquery
-            ci_credentials destination-postgres
-            ci_credentials destination-snowflake
+            ci_credentials destination-bigquery write-to-storage
+            ci_credentials destination-postgres write-to-storage
+            ci_credentials destination-snowflake write-to-storage
           fi
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
@@ -131,6 +131,18 @@ jobs:
           command: ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }}
           attempt_limit: 3
           attempt_delay: 10000 # in ms
+      - name: Update Integration Test Credentials after test run for ${{ github.event.inputs.connector }}
+        run: |
+          source venv/bin/activate
+          ci_credentials ${{ github.event.inputs.connector }} update-secrets
+          # normalization also runs destination-specific tests, so fetch their creds also
+          if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then
+            ci_credentials destination-bigquery update-secrets
+            ci_credentials destination-postgres update-secrets
+            ci_credentials destination-snowflake update-secrets
+          fi
+        env:
+          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
       - name: Archive test reports artifacts
         if: github.event.inputs.comment-id && failure()
         uses: actions/upload-artifact@v3

.github/workflows/test-performance-command.yml

@@ -107,12 +107,12 @@ jobs:
       - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }}
         run: |
           source venv/bin/activate
-          ci_credentials ${{ github.event.inputs.connector }}
+          ci_credentials ${{ github.event.inputs.connector }} write-to-storage
           # normalization also runs destination-specific tests, so fetch their creds also
           if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then
-            ci_credentials destination-bigquery
-            ci_credentials destination-postgres
-            ci_credentials destination-snowflake
+            ci_credentials destination-bigquery write-to-storage
+            ci_credentials destination-postgres write-to-storage
+            ci_credentials destination-snowflake write-to-storage
           fi
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
@@ -124,6 +124,18 @@ jobs:
           ACTION_RUN_ID: ${{github.run_id}}
           # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners.
           TZ: UTC
+      - name: Update Integration Test Credentials after test run for ${{ github.event.inputs.connector }}
+        run: |
+          source venv/bin/activate
+          ci_credentials ${{ github.event.inputs.connector }} update-secrets
+          # normalization also runs destination-specific tests, so fetch their creds also
+          if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then
+            ci_credentials destination-bigquery update-secrets
+            ci_credentials destination-postgres update-secrets
+            ci_credentials destination-snowflake update-secrets
+          fi
+        env:
+          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
       - name: Archive test reports artifacts
         if: github.event.inputs.comment-id && failure()
         uses: actions/upload-artifact@v3

tools/ci_credentials/README.md

@@ -1,6 +1,8 @@
 # CI Credentials
+CLI tooling to read and manage GSM secrets:
+- `write-to-storage` download a connector's secrets locally in the connector's `secret` folder
+- `update-secrets` uploads new connector secret version that were locally updated.
 
-Connects to GSM to download connection details.
 
 ## Development
 
@@ -25,14 +27,22 @@ After making a change, you have to reinstall it to run the bash command: `pip in
 
 The `VERSION=dev` will make it so it knows to use your local current working directory and not the Github Action one.
 
-Pass in a connector name. For example:
 
+### Help
 ```bash
-VERSION=dev ci_credentials destination-snowflake
+ci_credentials --help
 ```
 
-To make sure it get's all changes every time, you can run this:
+### Write to storage
+To download GSM secrets to `airbyte-integrations/connectors/source-bings-ads/secrets`:
+```bash
+ci_credentials source-bing-ads write-to-storage
+```
+
+### Update secrets
+To upload to GSM newly updated configurations from `airbyte-integrations/connectors/source-bings-ads/secrets/updated_configurations`:
 
 ```bash
-pip install --quiet -e ./tools/ci_* && VERSION=dev ci_credentials destination-snowflake
-```
+ci_credentials source-bing-ads update-secrets
+```
+

tools/ci_credentials/ci_credentials/__init__.py

@@ -1,6 +1,6 @@
-# from .main import main
-from .secrets_loader import SecretsLoader
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+from .secrets_manager import SecretsManager
 
-__all__ = (
-    "SecretsLoader",
-)
+__all__ = ("SecretsManager",)

tools/ci_credentials/ci_credentials/main.py

@@ -1,43 +1,62 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
 import json
-import os
 import sys
 from json.decoder import JSONDecodeError
 
+import click
 from ci_common_utils import Logger
-from . import SecretsLoader
+
+from . import SecretsManager
 
 logger = Logger()
 
 ENV_GCP_GSM_CREDENTIALS = "GCP_GSM_CREDENTIALS"
 
 
 # credentials of GSM and GitHub secrets should be shared via shell environment
-
-def main() -> int:
-    if len(sys.argv) != 2:
-        return logger.error("uses one script argument only: <unique connector name>")
-
+@click.group()
+@click.argument("connector_name")
+@click.option("--gcp-gsm-credentials", envvar="GCP_GSM_CREDENTIALS")
+@click.pass_context
+def ci_credentials(ctx, connector_name: str, gcp_gsm_credentials):
+    ctx.ensure_object(dict)
+    ctx.obj["connector_name"] = connector_name
     # parse unique connector name, because it can have the common prefix "connectors/<unique connector name>"
-    connector_name = sys.argv[1].split("/")[-1]
+    connector_name = connector_name.split("/")[-1]
     if connector_name == "all":
         # if needed to load all secrets
         connector_name = None
 
     # parse GCP_GSM_CREDENTIALS
     try:
-        gsm_credentials = json.loads(os.getenv(ENV_GCP_GSM_CREDENTIALS) or "{}")
+        gsm_credentials = json.loads(gcp_gsm_credentials) if gcp_gsm_credentials else {}
     except JSONDecodeError as e:
         return logger.error(f"incorrect GCP_GSM_CREDENTIALS value, error: {e}")
 
     if not gsm_credentials:
         return logger.error("GCP_GSM_CREDENTIALS shouldn't be empty!")
 
-    loader = SecretsLoader(
+    secret_manager = SecretsManager(
         connector_name=connector_name,
         gsm_credentials=gsm_credentials,
     )
-    return loader.write_to_storage(loader.read_from_gsm())
+    ctx.obj["secret_manager"] = secret_manager
+    ctx.obj["connector_secrets"] = secret_manager.read_from_gsm()
+
+
+@ci_credentials.command(help="Download GSM secrets locally to the connector's secrets directory.")
+@click.pass_context
+def write_to_storage(ctx):
+    return ctx.obj["secret_manager"].write_to_storage(ctx.obj["connector_secrets"])
+
+
+@ci_credentials.command(help="Update GSM secrets according to the content of the secrets/updated_configurations directory.")
+@click.pass_context
+def update_secrets(ctx):
+    return ctx.obj["secret_manager"].update_secrets(ctx.obj["connector_secrets"])
 
 
-if __name__ == '__main__':
-    sys.exit(main())
+if __name__ == "__main__":
+    sys.exit(ci_credentials(obj={}))

tools/ci_credentials/ci_credentials/models.py

@@ -0,0 +1,56 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+from __future__ import (  # Used to evaluate type hints at runtime, a NameError: name 'ConfigObserver' is not defined is thrown otherwise
+    annotations,
+)
+
+from dataclasses import dataclass
+
+DEFAULT_SECRET_FILE = "config"
+
+
+@dataclass
+class Secret:
+    connector_name: str
+    configuration_file_name: str
+    value: str
+
+    @property
+    def name(self) -> str:
+        return self.generate_secret_name(self.connector_name, self.configuration_file_name)
+
+    @staticmethod
+    def generate_secret_name(connector_name: str, configuration_file_name: str) -> str:
+        """
+        Generates an unique GSM secret name.
+        Format of secret name: SECRET_<CAPITAL_CONNECTOR_NAME>_<OPTIONAL_UNIQUE_FILENAME_PART>__CREDS
+        Examples:
+            1. connector_name: source-linnworks, filename: dsdssds_a-b---_---_config.json
+                => SECRET_SOURCE-LINNWORKS_DSDSSDS_A-B__CREDS
+            2. connector_name: source-s3, filename: config.json
+                => SECRET_SOURCE-LINNWORKS__CREDS
+        """
+        name_parts = ["secret", connector_name]
+        filename_wo_ext = configuration_file_name.replace(".json", "")
+        if filename_wo_ext != DEFAULT_SECRET_FILE:
+            name_parts.append(filename_wo_ext.replace(DEFAULT_SECRET_FILE, "").strip("_-"))
+        name_parts.append("_creds")
+        return "_".join(name_parts).upper()
+
+    @property
+    def directory(self) -> str:
+        if self.connector_name == "base-normalization":
+            return f"airbyte-integrations/bases/{self.connector_name}/secrets"
+        else:
+            return f"airbyte-integrations/connectors/{self.connector_name}/secrets"
+
+
+@dataclass
+class RemoteSecret(Secret):
+    enabled_version: str
+
+    @classmethod
+    def from_secret(cls, secret: Secret, enabled_version: str) -> RemoteSecret:
+        return RemoteSecret(secret.connector_name, secret.configuration_file_name, secret.value, enabled_version)