Create module and entrypoint for gcp (#13)

.github/workflows/publish.yml

@@ -17,18 +17,15 @@ jobs:
         shell: bash
 
     steps:
-      # Checkout the repository to the GitHub Actions runner
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Setup Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: '1.18'
-
       - name: Set up Nullstone
         uses: nullstone-io/setup-nullstone-action@v0
 
+      - name: Setup Go
+        uses: actions/setup-go@v4
+
       - name: Build binaries into packages
         run: |
           export PATH=$(go env GOPATH)/bin:$PATH
@@ -37,12 +34,19 @@ jobs:
 
       - name: Find version
         id: version
-        run: echo ::set-output name=tag::${GITHUB_REF#refs/tags/v}
+        run: echo "MODULE_VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV
 
-      - id: publish
-        name: Publish
-        working-directory: aws/tf/
+      - id: publish-aws
+        name: Publish AWS
+        working-directory: aws/tf
+        run: |
+          nullstone modules publish \
+            --include=files/pg-db-admin.zip \
+            --version=${{ env.MODULE_VERSION }}
+      - id: publish-gcp
+        name: Publish GCP
+        working-directory: gcp/tf
         run: |
           nullstone modules publish \
-            --version=${{ steps.version.outputs.tag }} \
-            --include=files/pg-db-admin.zip
+            --include=files/pg-db-admin.zip \
+            --version=${{ env.MODULE_VERSION }}

Makefile

@@ -8,9 +8,17 @@ tools:
 build:
 	mkdir -p ./aws/tf/files
 	GOOS=linux GOARCH=amd64 go build -o ./aws/tf/files/pg-db-admin ./aws/
+	# Run build on gcp to ensure a successful build, we discard it
+	GOOS=linux GOARCH=amd64 go build -o ./gcp/tf/files/pg-db-admin ./gcp/; rm -f ./gcp/tf/files/pg-db-admin
 
 package: tools
+	# Package aws module using build-lambda-zip which produces a viable package from any OS
 	cd ./aws/tf && build-lambda-zip --output files/pg-db-admin.zip files/pg-db-admin
+	# Package gcp module (source code instead of binary)
+	# For GCP, main.go *must* be in the root of the zip file
+	cp gcp/main.go main.go && \
+		zip -r gcp/tf/files/pg-db-admin.zip go.mod go.sum main.go ./api/ ./postgresql/ ./vendor/; \
+		rm main.go
 
 acc: acc-up acc-run acc-down
 

gcp/main.go

@@ -0,0 +1,30 @@
+package pg_db_admin
+
+// This is the entrypoint for a GCP Cloud Function
+// A Cloud Function (2nd gen) *must* be built using GCP Cloud Build
+// This requires us to do the following:
+//   - Package all source code (including vendor) in the zip file
+//   - main.go *must* be at the root of the zip file
+//   - package name in main.go must match module name defined in go.mod (cannot be `package main`)
+//
+// This entrypoint does not run code; it only registers a trigger that is used by the runtime upon execution
+
+import (
+	"fmt"
+	_ "github.com/GoogleCloudPlatform/functions-framework-go/funcframework"
+	"github.com/GoogleCloudPlatform/functions-framework-go/functions"
+	"github.com/nullstone-modules/pg-db-admin/api"
+	"github.com/nullstone-modules/pg-db-admin/postgresql"
+	"os"
+)
+
+var (
+	dbConnUrlEnvVar = "DB_CONN_URL"
+)
+
+func init() {
+	fmt.Println("Initializing pg-db-admin...")
+	store := postgresql.NewStore(os.Getenv(dbConnUrlEnvVar))
+	router := api.CreateRouter(store)
+	functions.HTTP("pg-db-admin", router.ServeHTTP)
+}

gcp/tf/.nullstone/module.yml

@@ -0,0 +1,13 @@
+org_name: nullstone
+name: gcp-pg-db-admin
+friendly_name: GCP Cloud Function for PostgreSQL Admin
+description: Creates a Google Cloud Function to administer a postgresql database without the use of SSH Tunnel or VPN
+category: block
+subcategory: ""
+provider_types:
+    - gcp
+platform: postgres
+subplatform: ""
+type: ""
+appCategories: []
+is_public: true

gcp/tf/connection_url.tf

@@ -0,0 +1,14 @@
+resource "google_secret_manager_secret" "db_admin_pg" {
+  secret_id = "${var.name}_conn_url"
+  labels    = var.labels
+
+  replication {
+    automatic = true
+  }
+}
+
+resource "google_secret_manager_secret_version" "db_admin_pg" {
+  secret      = google_secret_manager_secret.db_admin_pg.id
+  secret_data = "postgres://${urlencode(var.username)}:${urlencode(var.password)}@${var.host}:${var.port}/${urlencode(var.database)}"
+  enabled     = true
+}

gcp/tf/executor.tf

@@ -0,0 +1,22 @@
+locals {
+  truncated_executor_len = min(length(var.name), 28 - length("executor-"))
+  executor_name  = "executor-${substr(var.name, 0, local.truncated_executor_len)}"
+}
+
+resource "google_service_account" "executor" {
+  account_id   = local.executor_name
+  display_name = "Executor for pg db admin ${var.name}"
+}
+
+resource "google_project_iam_member" "executor_artifacts" {
+  project = local.project_id
+  role    = "roles/artifactregistry.reader"
+  member  = "serviceAccount:${google_service_account.executor.email}"
+}
+
+resource "google_secret_manager_secret_iam_member" "executor_secrets" {
+  project   = local.project_id
+  secret_id = google_secret_manager_secret.db_admin_pg.secret_id
+  role      = "roles/secretmanager.secretAccessor"
+  member    = "serviceAccount:${google_service_account.executor.email}"
+}

gcp/tf/function.tf

@@ -0,0 +1,42 @@
+resource "google_cloudfunctions2_function" "function" {
+  name        = var.name
+  location    = local.region
+  description = "${var.name} Postgresql DB Admin"
+  labels      = var.labels
+
+  build_config {
+    runtime     = "go120"
+    entry_point = "pg-db-admin"
+
+    environment_variables = {
+      "SOURCE_HASH" : google_storage_bucket_object.binary.detect_md5hash
+    }
+
+    source {
+      storage_source {
+        bucket = google_storage_bucket.binaries.name
+        object = google_storage_bucket_object.binary.name
+      }
+    }
+  }
+
+  service_config {
+    service_account_email            = google_service_account.executor.email
+    available_cpu                    = "2"
+    available_memory                 = "512Mi"
+    timeout_seconds                  = 20
+    max_instance_count               = 100
+    max_instance_request_concurrency = 50
+    all_traffic_on_latest_revision   = true
+    ingress_settings                 = "ALLOW_ALL"
+    vpc_connector_egress_settings    = "ALL_TRAFFIC"
+    vpc_connector                    = var.vpc_access_connector_name
+
+    secret_environment_variables {
+      key        = "DB_CONN_URL"
+      project_id = local.project_id
+      secret     = google_secret_manager_secret.db_admin_pg.secret_id
+      version    = google_secret_manager_secret_version.db_admin_pg.version
+    }
+  }
+}

gcp/tf/gcp.tf

@@ -0,0 +1,7 @@
+data "google_client_config" "this" {}
+
+locals {
+  project_id    = data.google_client_config.this.project
+  region        = data.google_client_config.this.region
+  region_prefix = lower(substr(local.region, 0, 2))
+}

gcp/tf/invoker.tf

@@ -0,0 +1,19 @@
+locals {
+  truncated_len = min(length(var.name), 28 - length("invoker-"))
+  invoker_name  = "invoker-${substr(var.name, 0, local.truncated_len)}"
+}
+
+resource "google_service_account" "invoker" {
+  account_id   = local.invoker_name
+  display_name = "Invoker for pg db admin ${var.name}"
+}
+
+resource "google_service_account_key" "invoker" {
+  service_account_id = google_service_account.invoker.account_id
+}
+
+resource "google_project_iam_member" "invoker_basic" {
+  project = local.project_id
+  role    = "roles/run.invoker"
+  member  = "serviceAccount:${google_service_account.invoker.email}"
+}

gcp/tf/outputs.tf

@@ -0,0 +1,17 @@
+output "function_name" {
+  value = google_cloudfunctions2_function.function.name
+}
+
+output "function_url" {
+  value = try(google_cloudfunctions2_function.function.service_config[0].uri, "")
+}
+
+output "invoker" {
+  value = {
+    email       = google_service_account.invoker.email
+    private_key = google_service_account_key.invoker.private_key
+  }
+
+  description = "object({ email: string, private_key: string }) ||| A GCP service account with explicit privilege invoke db admin cloud function."
+  sensitive   = true
+}

gcp/tf/storage.tf

@@ -0,0 +1,17 @@
+locals {
+  storage_location = local.region_prefix == "us" ? "US" : (local.region_prefix == "eu" ? "EU" : "ASIA")
+  package_filename = "${path.module}/files/pg-db-admin.zip"
+}
+
+resource "google_storage_bucket" "binaries" {
+  name          = "${var.name}-binaries"
+  location      = local.storage_location
+  labels        = var.labels
+  force_destroy = true
+}
+
+resource "google_storage_bucket_object" "binary" {
+  bucket = google_storage_bucket.binaries.name
+  name   = "pg-db-admin.zip"
+  source = local.package_filename
+}

gcp/tf/variables.tf

@@ -0,0 +1,44 @@
+variable "name" {
+  description = "The name of the cloud function function and "
+  type        = string
+}
+
+variable "labels" {
+  description = "A map of labels that are applied to GCP resources"
+  type        = map(string)
+}
+
+variable "host" {
+  description = "The database cluster host to connect"
+  type        = string
+}
+
+variable "port" {
+  description = "The database cluster port to connect"
+  type        = string
+  default     = "3306"
+}
+
+variable "database" {
+  description = "The initial database to connect"
+  type        = string
+  default     = ""
+}
+
+variable "username" {
+  description = "Postgresql username"
+  type        = string
+}
+
+variable "password" {
+  description = "Postgresql password"
+  type        = string
+}
+
+variable "vpc_access_connector_name" {
+  type        = string
+  description = <<EOF
+This module requires a VPC Serverless Access Connector to reach the Cloud SQL instance in a private network.
+This variable configures the function to use an existing access connector.
+EOF
+}

go.mod

@@ -1,8 +1,9 @@
 module github.com/nullstone-modules/pg-db-admin
 
-go 1.18
+go 1.20
 
 require (
+	github.com/GoogleCloudPlatform/functions-framework-go v1.7.4
 	github.com/aws/aws-lambda-go v1.34.1
 	github.com/aws/aws-sdk-go-v2 v1.7.1
 	github.com/aws/aws-sdk-go-v2/config v1.5.0
@@ -13,18 +14,27 @@ require (
 	github.com/lib/pq v1.10.2
 	github.com/nullstone-io/go-lambda-api-sdk v0.0.0-20220829133353-4a8c1d845640
 	github.com/nullstone-io/go-rest-api v0.0.0-20220913221656-e752d22f9894
-	github.com/stretchr/testify v1.8.0
+	github.com/stretchr/testify v1.8.1
 )
 
 require (
+	cloud.google.com/go/functions v1.13.0 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.3.1 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.3.0 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.1.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.3.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.6.0 // indirect
 	github.com/aws/smithy-go v1.6.0 // indirect
+	github.com/cloudevents/sdk-go/v2 v2.14.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/json-iterator/go v1.1.10 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
+	github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	go.uber.org/atomic v1.4.0 // indirect
+	go.uber.org/multierr v1.1.0 // indirect
+	go.uber.org/zap v1.10.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )