feat(ci): nightly checks with connectivity recovery test (#1471)

Resolves #1465

---------

Co-authored-by: Ryan Schumacher <jschumacher@virtru.com>

.github/scripts/connectivity-test.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# Randomly drop db connections to test CLI connectivity for 15 minutes total
+start_time=$(date +%s)
+postgresql_container_id=$(docker ps --filter "name=platform-opentdfdb-1" -q)
+
+resource_subcommands=("attributes" "attributes namespaces" "subject-mappings" "resource-mappings" "kas-registry")
+
+while true; do
+    # Randomly wait before running the connectivity test (between 1 and 10 seconds)
+    sleep $((RANDOM % 10 + 1))
+
+    echo "Restarting PostgreSQL container..."
+    docker restart $postgresql_container_id
+
+    # Determine how many random otdfctl commands to run after the restart
+    num_runs=$((RANDOM % 5 + 1))  # Randomly choose to run between 1 and 5 times
+
+    for ((i=0; i<num_runs; i++)); do
+        random_subcommand=${resource_subcommands[$RANDOM % ${#resource_subcommands[@]}]}
+
+        # Introduce random delay before each execution (between 1 and 4 seconds)
+        sleep $((RANDOM % 4 + 1))
+
+        echo "Running randomly selected command './otdfctl policy $random_subcommand list...'"
+        result=$(./otdfctl policy $random_subcommand list --with-client-creds '{"clientId":"opentdf","clientSecret":"secret"}' --host http://localhost:8080 | grep -i "success")
+        echo $result
+        if [ -z "$result" ]; then
+        echo "Failure: 'success' not found in output; CLI failed."
+        exit 1
+        fi
+    done
+    # Exit if 15 minutes have passed (900 seconds)
+    current_time=$(date +%s)
+    elapsed_time=$((current_time - start_time))
+
+    if [ $elapsed_time -ge 120 ]; then 
+    # if [ $elapsed_time -ge 900 ]; then 
+        exit 0
+    fi
+done

.github/workflows/checks.yaml

@@ -17,6 +17,7 @@ on:
       - main
     types:
       - checks_requested
+  workflow_call:
 
 jobs:
   go:

.github/workflows/nightly-checks.yaml

@@ -0,0 +1,76 @@
+name: 'Nightly Checks'
+
+on:
+  schedule:
+    # Run at 12:15 AM UTC (Scheduled actions are not guaranteed during times of high load like the top of the
+    # hour or 00:00. See discussion: https://github.com/orgs/community/discussions/27130)
+    - cron: '15 0 * * *'
+
+jobs:
+  db-flakiness-recovery:
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+    steps:
+      ######## CHECKOUT/SETUP PLATFORM #############
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+        with:
+          fetch-depth: 0
+          path: platform
+      - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7
+        with:
+          go-version-file: 'platform/service/go.mod'
+          check-latest: false
+          cache-dependency-path: |
+            platform/examples/go.sum
+            platform/protocol/go/go.sum
+            platform/sdk/go.sum
+            platform/service/go.sum
+      ######## SPIN UP PLATFORM/BACKEND #############
+      - run: |
+          ./.github/scripts/init-temp-keys.sh
+          cp opentdf-dev.yaml opentdf.yaml
+        working-directory: platform
+      - name: Added Trusted Certs
+        run: |
+          sudo chmod -R 777 ./keys
+          sudo apt-get install -y ca-certificates
+          sudo cp ./keys/localhost.crt /usr/local/share/ca-certificates
+          sudo update-ca-certificates
+        working-directory: platform
+      - run: docker compose up -d --wait --wait-timeout 240
+        working-directory: platform
+      - run: go run ./service provision keycloak
+        working-directory: platform
+      - run: go run ./service provision fixtures
+        working-directory: platform
+      - uses: JarvusInnovations/background-action@2428e7b970a846423095c79d43f759abf979a635
+        name: start server in background
+        with:
+          run: >
+            go build -o opentdf -v service/main.go
+            && .github/scripts/watch.sh opentdf.yaml ./opentdf start
+          wait-on: |
+            tcp:localhost:8080
+          log-output-if: true
+          wait-for: 90s
+          working-directory: platform
+
+      ######## CHECKOUT/BUILD 'otdfctl' #############
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+        with:
+          repository: opentdf/otdfctl
+          ref: main
+          fetch-depth: 0
+          path: otdfctl
+      - run: go build -o otdfctl
+        working-directory: otdfctl
+      - run: cp otdfctl ../platform
+        working-directory: otdfctl
+
+      ######## RUN TESTS #############
+      - run: ./.github/scripts/connectivity-test.sh
+        name: Flaky Connectivity Test
+        working-directory: platform
+  ci-checks:
+    uses: opentdf/platform/.github/workflows/checks.yaml@main