Clean up cicd (bcgov#1547)

* clean up cicd

.github/workflows/emailer-cd-gcp.yml

@@ -0,0 +1,34 @@
+name: Namex Emailer CD GCP
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "services/emailer/**"
+  workflow_dispatch:
+    inputs:
+      target:
+        description: "Deploy To"
+        required: true
+        type: choice
+        options:
+          - dev
+          - test
+          - sandbox
+          - prod
+
+jobs:
+  namex-emailer-cd:
+    permissions:
+      id-token: write
+      contents: write
+
+    uses: bcgov/bcregistry-sre/.github/workflows/cloud-run-service-cd.yaml@main
+    with:
+      target: ${{ github.event.inputs.target }}  # Corrected the input reference
+      app_name: "namex-emailer"
+      working_directory: "./services/emailer"
+    secrets:
+      WORKLOAD_IDENTIFY_POOLS_PROVIDER: ${{ secrets.WORKLOAD_IDENTIFY_POOLS_PROVIDER }}
+      GCP_SERVICE_ACCOUNT: ${{ secrets.GCP_SERVICE_ACCOUNT }}

api/.env.sample

@@ -71,8 +71,8 @@ JWT_OIDC_CACHING_ENABLED=True
 JWT_OIDC_JWKS_CACHE_TIMEOUT=300
 
 # PUBSUB
-GCP_AUTH_KEY=
-EMAILER_TOPIC=
+BUSINESS_GCP_AUTH_KEY=
+NAMEX_MAILER_TOPIC=
 NAMEX_NR_STATE_TOPIC=
 
 # MARS

api/Makefile

@@ -24,6 +24,7 @@ setup: clean install ## Setup the project
 
 clean: clean-build clean-pyc clean-test ## Clean the project
 	rm -rf .venv/
+	rm -rf poetry.lock
 
 clean-build: ## Clean build files
 	rm -fr build/

api/config.py

@@ -89,7 +89,6 @@ class Config(object):
     JWT_OIDC_JWKS_CACHE_TIMEOUT = int(os.getenv('JWT_OIDC_JWKS_CACHE_TIMEOUT', '300'))
 
     JWT_OIDC_AUDIENCE = os.getenv('JWT_OIDC_AUDIENCE')
-    JWT_OIDC_CLIENT_SECRET = os.getenv('JWT_OIDC_CLIENT_SECRET')
 
     SBC_SVC_AUTH_URL = os.getenv('KEYCLOAK_AUTH_TOKEN_URL', '')
     SBC_SVC_AUTH_CLIENT_ID = os.getenv('NAMEX_SBC_SERVICE_ACCOUNT_CLIENT_ID', '')
@@ -107,9 +106,9 @@ class Config(object):
     DISABLE_NAMEREQUEST_SOLR_UPDATES = int(os.getenv('DISABLE_NAMEREQUEST_SOLR_UPDATES', 0))
 
 
-    GCP_AUTH_KEY = os.getenv("GCP_AUTH_KEY", None)
-    NAMEX_NR_STATE_TOPIC = os.getenv("NAMEX_NR_STATE_TOPIC", "namex-pay-dev")
-    EMAILER_TOPIC = os.getenv("EMAILER_TOPIC", "namex-receipt-dev")
+    GCP_AUTH_KEY = os.getenv('BUSINESS_GCP_AUTH_KEY', None)
+    NAMEX_NR_STATE_TOPIC = os.getenv('NAMEX_NR_STATE_TOPIC', '')
+    EMAILER_TOPIC = os.getenv('NAMEX_MAILER_TOPIC', '')
 
     AUDIENCE = os.getenv("AUDIENCE", "https://pubsub.googleapis.com/google.pubsub.v1.Subscriber")
     PUBLISHER_AUDIENCE = os.getenv("PUBLISHER_AUDIENCE", "https://pubsub.googleapis.com/google.pubsub.v1.Publisher")
@@ -154,7 +153,7 @@ class TestConfig(Config):
         port=int(DB_PORT),
         name=DB_NAME
     )
-    EMAILER_TOPIC = os.getenv("EMAILER_TOPIC", "namex-receipt-dev")
+    EMAILER_TOPIC = os.getenv('NAMEX_MAILER_TOPIC', '')
 
     # We can't run NRO locally for running our tests
     DISABLE_NAMEREQUEST_NRO_UPDATES = int(os.getenv('DISABLE_NAMEREQUEST_NRO_UPDATES', 1))

jobs/nr-day-job/Makefile

@@ -16,6 +16,7 @@ setup: install ## Setup the project
 
 clean: clean-build clean-pyc clean-test ## Clean the project
 	rm -rf .venv/
+	rm -rf poetry.lock
 
 clean-build: ## Clean build files
 	rm -fr build/

jobs/nr-day-job/config.py

@@ -74,8 +74,8 @@ class Config():  # pylint: disable=too-few-public-methods
         name=DB_NAME,
     )
 
-    GCP_AUTH_KEY = os.getenv('GCP_AUTH_KEY', None)
-    EMAILER_TOPIC = os.getenv('EMAILER_TOPIC', '')
+    GCP_AUTH_KEY = os.getenv('BUSINESS_GCP_AUTH_KEY', None)
+    EMAILER_TOPIC = os.getenv('NAMEX_MAILER_TOPIC', '')
     NAMEX_NR_STATE_TOPIC = os.getenv('NAMEX_NR_STATE_TOPIC', '')
 
 class DevConfig(Config):  # pylint: disable=too-few-public-methods

jobs/nro-extractor/Makefile

@@ -22,6 +22,7 @@ setup: install ## Setup the project
 
 clean: clean-build clean-pyc clean-test ## Clean the project
 	rm -rf .venv/
+	rm -rf poetry.lock
 
 clean-build: ## Clean build files
 	rm -fr build/

jobs/nro-extractor/config.py

@@ -38,9 +38,9 @@ class BaseConfig(object):
     NRO_HOST = os.getenv('NRO_HOST', '')
     NRO_PORT = int(os.getenv('NRO_PORT', '1521'))
 
-    GCP_AUTH_KEY = os.getenv('GCP_AUTH_KEY', None)
+    GCP_AUTH_KEY = os.getenv('BUSINESS_GCP_AUTH_KEY', None)
     NAMEX_NR_STATE_TOPIC = os.getenv('NAMEX_NR_STATE_TOPIC', '')
-    EMAILER_TOPIC = os.getenv('EMAILER_TOPIC', '')
+    EMAILER_TOPIC = os.getenv('NAMEX_MAILER_TOPIC', '')
 
 
 class Config(BaseConfig):

services/emailer/config.py

@@ -102,7 +102,7 @@ class Config:  # pylint: disable=too-few-public-methods
     ACCOUNT_SVC_CLIENT_SECRET = os.getenv("KEYCLOAK_CLIENT_SECRET")
     ACCOUNT_SVC_TIMEOUT = os.getenv("KEYCLOAK_TIMEOUT")
 
-    SUB_AUDIENCE = os.getenv('SUB_AUDIENCE', '')
+    SUB_AUDIENCE = os.getenv('EMAILER_SUB_AUDIENCE', '')
     SUB_SERVICE_ACCOUNT = os.getenv('SUB_SERVICE_ACCOUNT', '')
 
     NAME_REQUEST_URL = os.getenv("NAME_REQUEST_URL", "")

services/emailer/devops/gcp/clouddeploy-targets.yaml

@@ -0,0 +1,100 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: deploy.cloud.google.com/v1
+kind: Target
+metadata:
+ name: dev
+description: Dev Environment
+deployParameters:
+  deploy-env: "development"
+  deploy-project-id: "a083gt-dev"
+  service-name: "namex-emailer-dev"
+  container-name: "namex-emailer-dev"
+  app-env: "dev"
+  service-account: "sa-api@a083gt-dev.iam.gserviceaccount.com"
+run:
+ location: projects/a083gt-dev/locations/northamerica-northeast1
+executionConfigs:
+- usages: [DEPLOY, RENDER]
+  artifactStorage: 'gs://c4hnrd-tools_clouddeploy/history'
+---
+
+apiVersion: deploy.cloud.google.com/v1
+kind: Target
+metadata:
+ name: test
+description: Test Environment
+deployParameters:
+  deploy-env: "development"
+  deploy-project-id: "a083gt-test"
+  service-name: "namex-emailer-test"
+  container-name: "namex-emailer-test"
+  app-env: "test"
+  service-account: "sa-api@a083gt-test.iam.gserviceaccount.com"
+run:
+ location: projects/a083gt-test/locations/northamerica-northeast1
+executionConfigs:
+- usages: [DEPLOY, RENDER]
+  artifactStorage: 'gs://c4hnrd-tools_clouddeploy/history'
+---
+
+apiVersion: deploy.cloud.google.com/v1
+kind: Target
+metadata:
+ name: sandbox
+description: Sandbox Environment
+requireApproval: true
+deployParameters:
+  deploy-env: "production"
+  deploy-project-id: "a083gt-tools"
+  service-name: "namex-emailer-sandbox"
+  container-name: "namex-emailer-sandbox"
+  app-env: "sandbox"
+  service-account: "sa-api@a083gt-tools.iam.gserviceaccount.com"
+  max-scale: "50"
+  container-concurrency: "20"
+  container-port: "8080"
+  resources-cpu: 4000m
+  resources-memory: 8Gi
+run:
+ location: projects/a083gt-tools/locations/northamerica-northeast1
+executionConfigs:
+- usages: [DEPLOY, RENDER]
+  artifactStorage: 'gs://c4hnrd-tools_clouddeploy/history'
+---
+
+apiVersion: deploy.cloud.google.com/v1
+kind: Target
+metadata:
+ name: prod
+description: Production Environment
+requireApproval: true
+deployParameters:
+  deploy-env: "production"
+  deploy-project-id: "a083gt-prod"
+  service-name: "namex-emailer-prod"
+  container-name: "namex-emailer-prod"
+  app-env: "production"
+  service-account: "sa-api@a083gt-prod.iam.gserviceaccount.com"
+  max-scale: "50"
+  container-concurrency: "20"
+  container-port: "8080"
+  resources-cpu: 4000m
+  resources-memory: 8Gi
+run:
+ location: projects/a083gt-prod/locations/northamerica-northeast1
+executionConfigs:
+- usages: [DEPLOY, RENDER]
+  artifactStorage: 'gs://c4hnrd-tools_clouddeploy/history'

services/emailer/devops/vaults.gcp.env

@@ -13,5 +13,5 @@ NAMEX_API_VERSION="op://API/$APP_ENV/namex-api/NAMEX_API_VERSION"
 AUDIENCE="op://gcp-queue/$APP_ENV/payment/AUDIENCE"
 TOPIC_NAME="op://gcp-queue/$APP_ENV/payment/TOPIC_NAME"
 PUBLISHER_AUDIENCE="op://gcp-queue/$APP_ENV/payment/PUBLISHER_AUDIENCE"
-GCP_AUTH_KEY="op://gcp-queue/$APP_ENV/payment/GCP_AUTH_KEY"
+GCP_AUTH_KEY="op://gcp-queue/$APP_ENV/payment/BUSINESS_GCP_AUTH_KEY"
 SENTRY_DSN=""

services/namex-pay/.env.sample

@@ -28,8 +28,8 @@ ORACLE_PORT=1521
 PAY_API_URL=
 PAY_API_VERSION=
 
-GCP_AUTH_KEY=
-EMAILER_TOPIC=
-SUB_AUDIENCE=
-SUB_SERVICE_ACCOUNT=
+BUSINESS_GCP_AUTH_KEY=
+NAMEX_MAILER_TOPIC=
+PAY_SUB_AUDIENCE=
+AUTHPAY_SERVICE_ACCOUNT=
 DEBUG_REQUEST=

services/namex-pay/Makefile

@@ -16,6 +16,7 @@ setup: install ## Setup the project
 
 clean: clean-build clean-pyc clean-test ## Clean the project
 	rm -rf .venv/
+	rm -rf poetry.lock
 
 clean-build: ## Clean build files
 	rm -fr build/

services/namex-pay/config.py

@@ -82,13 +82,13 @@ class Config():  # pylint: disable=too-few-public-methods
     NRO_HOST = os.getenv('ORACLE_HOST', '')
     NRO_PORT = int(os.getenv('ORACLE_PORT', '1521'))
 
-    GCP_AUTH_KEY = os.getenv('GCP_AUTH_KEY', None)
-    EMAILER_TOPIC = os.getenv('EMAILER_TOPIC', '')
+    GCP_AUTH_KEY = os.getenv('BUSINESS_GCP_AUTH_KEY', None)
+    EMAILER_TOPIC = os.getenv('NAMEX_MAILER_TOPIC', '')
     NAMEX_NR_STATE_TOPIC = os.getenv('NAMEX_NR_STATE_TOPIC', '')
     AUDIENCE = os.getenv('AUDIENCE', 'https://pubsub.googleapis.com/google.pubsub.v1.Subscriber')
     PUBLISHER_AUDIENCE = os.getenv('PUBLISHER_AUDIENCE', 'https://pubsub.googleapis.com/google.pubsub.v1.Publisher')
-    SUB_AUDIENCE = os.getenv('SUB_AUDIENCE', '')
-    SUB_SERVICE_ACCOUNT = os.getenv('SUB_SERVICE_ACCOUNT', '')
+    SUB_AUDIENCE = os.getenv('PAY_SUB_AUDIENCE', '')
+    SUB_SERVICE_ACCOUNT = os.getenv('AUTHPAY_SERVICE_ACCOUNT', '')
     DEBUG_REQUEST = os.getenv('DEBUG_REQUEST', False)
 
     ENVIRONMENT = os.getenv('ENVIRONMENT', 'prod')
@@ -116,7 +116,7 @@ class TestConfig(Config):  # pylint: disable=too-few-public-methods
     DB_HOST = os.getenv('DATABASE_TEST_HOST', '127.0.0.1')
     DB_PORT = os.getenv('DATABASE_TEST_PORT', '5432')
     SQLALCHEMY_DATABASE_URI = f'postgresql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}'
-    EMAILER_TOPIC = os.getenv("EMAILER_TOPIC", "namex-receipt-dev")
+    EMAILER_TOPIC = os.getenv('NAMEX_MAILER_TOPIC', '')
 
 class ProdConfig(Config):  # pylint: disable=too-few-public-methods
     """Production environment configuration."""

services/pubsub/tests/test_queue.py

@@ -104,7 +104,7 @@ def test_gcp_pubsub_connectivity():
     load_dotenv(".env")
 
     app = flask.Flask(__name__)
-    app.config['GCP_AUTH_KEY'] = os.getenv('GCP_AUTH_KEY')
+    app.config['GCP_AUTH_KEY'] = os.getenv('BUSINESS_GCP_AUTH_KEY')
 
     queue = GcpQueue()
     queue.init_app(app)

services/solr-names-updater/Makefile

@@ -16,6 +16,7 @@ setup: install ## Setup the project
 
 clean: clean-build clean-pyc clean-test ## Clean the project
 	rm -rf .venv/
+	rm -rf poetry.lock
 
 clean-build: ## Clean build files
 	rm -fr build/

services/solr-names-updater/config.py

@@ -82,13 +82,13 @@ class Config():  # pylint: disable=too-few-public-methods
         name=DB_NAME,
     )
 
-    GCP_AUTH_KEY = os.getenv('GCP_AUTH_KEY', None)
-    EMAILER_TOPIC = os.getenv('EMAILER_TOPIC', 'namex-receipt-dev')
+    GCP_AUTH_KEY = os.getenv('BUSINESS_GCP_AUTH_KEY', None)
+    EMAILER_TOPIC = os.getenv('NAMEX_MAILER_TOPIC', '')
     AUDIENCE = os.getenv('AUDIENCE', 'https://pubsub.googleapis.com/google.pubsub.v1.Subscriber')
     PUBLISHER_AUDIENCE = os.getenv('PUBLISHER_AUDIENCE', 'https://pubsub.googleapis.com/google.pubsub.v1.Publisher')
     DEBUG_REQUEST = os.getenv('DEBUG_REQUEST', False)
-    SUB_AUDIENCE = os.getenv('SUB_AUDIENCE', '')
-    SUB_SERVICE_ACCOUNT = os.getenv('SUB_SERVICE_ACCOUNT', '')
+    SUB_AUDIENCE = os.getenv('SOLR_SUB_AUDIENCE', '')
+    SUB_SERVICE_ACCOUNT = os.getenv('BUSINESS_SERVICE_ACCOUNT', '')
 
     SOLR_FEEDER_API_URL = os.getenv('SOLR_FEEDER_API_URL', None)
 
@@ -100,7 +100,6 @@ class Config():  # pylint: disable=too-few-public-methods
     JWT_OIDC_JWKS_URI = os.getenv('JWT_OIDC_JWKS_URI')
     JWT_OIDC_ISSUER = os.getenv('JWT_OIDC_ISSUER')
     JWT_OIDC_AUDIENCE = os.getenv('JWT_OIDC_AUDIENCE')
-    JWT_OIDC_CLIENT_SECRET = os.getenv('JWT_OIDC_CLIENT_SECRET')
     JWT_OIDC_CACHING_ENABLED = os.getenv('JWT_OIDC_CACHING_ENABLED')
 
     JWT_OIDC_JWKS_CACHE_TIMEOUT = int(os.getenv('JWT_OIDC_JWKS_CACHE_TIMEOUT', '300'))