diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml new file mode 100644 index 0000000..4516b2d --- /dev/null +++ b/.github/actions/deploy/action.yml @@ -0,0 +1,111 @@ +name: Deploy to GCP +description: Deploy to GCP +inputs: + project_id: + description: "The GCP project ID" + required: true + identity_provider: + description: "The identity provider for the workload identity" + required: true + service_account_email: + description: "The service account email" + required: true + storage_bucket: + description: "The GCP storage bucket" + required: true + +runs: + using: composite + steps: + - name: Set globals + id: globals + shell: bash + run: | + echo "MAIN_SCHEDULE_NAME=skidname_main" >> "${GITHUB_OUTPUT}" + echo "MAIN_SCHEDULE_CRON=0 22 * * 6" >> "${GITHUB_OUTPUT}" + echo "MAIN_SCHEDULE_DESCRIPTION=Trigger the skidname-skid bot every saturday evening at 10pm" >> "${GITHUB_OUTPUT}" + echo "VALIDATOR_SCHEDULE_NAME=validator" >> "${GITHUB_OUTPUT}" + echo "VALIDATOR_SCHEDULE_DESCRIPTION=Trigger the skidname validation bot every 1st of April, May, and June at 8am" >> "${GITHUB_OUTPUT}" + echo "VALIDATOR_SCHEDULE_CRON=0 8 1 4-6 *" >> "${GITHUB_OUTPUT}" + echo "TOPIC_NAME=skidname-topic" >> "${GITHUB_OUTPUT}" + + - name: ๐Ÿ—๏ธ Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v2 + with: + create_credentials_file: true + token_format: access_token + workload_identity_provider: ${{ inputs.identity_provider }} + service_account: ${{ inputs.service_account_email }} + + - name: ๐Ÿ“ฅ Create Main PubSub topic + shell: bash + run: | + if [ ! "$(gcloud pubsub topics list | grep ${{ steps.globals.outputs.TOPIC_NAME }})" ]; then + gcloud pubsub topics create ${{ steps.globals.outputs.TOPIC_NAME }} --quiet + fi + + - name: ๐Ÿš€ Deploy Cloud Function + id: deploy + uses: google-github-actions/deploy-cloud-functions@v3 + with: + name: skidname-skid + runtime: python311 + entry_point: subscribe + source_dir: src/skidname + service_account: cloud-function-sa@${{ inputs.project_id }}.iam.gserviceaccount.com + event_trigger_type: google.cloud.pubsub.topic.v1.messagePublished + event_trigger_pubsub_topic: projects/${{ inputs.project_id }}/topics/${{ steps.globals.outputs.TOPIC_NAME }} + memory: 1024M + service_timeout: 9m + environment_variables: STORAGE_BUCKET=${{ inputs.storage_bucket }} + secrets: | + /secrets/app/secrets.json=${{ inputs.project_id }}/skid-secrets + max_instance_count: 1 + event_trigger_retry: false + + - name: ๐Ÿ•ฐ๏ธ Create Main Cloud Scheduler + shell: bash + run: | + if [ ! "$(gcloud scheduler jobs list --location=us-central1 | grep ${{ steps.globals.outputs.MAIN_SCHEDULE_NAME }})" ]; then + gcloud scheduler jobs create pubsub "${{ steps.globals.outputs.MAIN_SCHEDULE_NAME }}" \ + --description="${{ steps.globals.outputs.MAIN_SCHEDULE_DESCRIPTION }}" \ + --schedule="${{ steps.globals.outputs.MAIN_SCHEDULE_CRON }}" \ + --time-zone=America/Denver \ + --location=us-central1 \ + --topic="${{ steps.globals.outputs.TOPIC_NAME }}" \ + --message-body='run the skid' \ + --quiet + else + gcloud scheduler jobs update pubsub "${{ steps.globals.outputs.MAIN_SCHEDULE_NAME }}" \ + --description="${{ steps.globals.outputs.MAIN_SCHEDULE_DESCRIPTION }}" \ + --schedule="${{ steps.globals.outputs.MAIN_SCHEDULE_CRON }}" \ + --time-zone=America/Denver \ + --location=us-central1 \ + --topic="${{ steps.globals.outputs.TOPIC_NAME }}" \ + --message-body='run the skid' \ + --quiet + fi + + - name: ๐Ÿ•ฐ๏ธ Create Validator Cloud Scheduler + shell: bash + run: | + if [ ! "$(gcloud scheduler jobs list --location=us-central1 | grep ${{ steps.globals.outputs.VALIDATOR_SCHEDULE_NAME }})" ]; then + gcloud scheduler jobs create pubsub "${{ steps.globals.outputs.VALIDATOR_SCHEDULE_NAME }}" \ + --description="${{ steps.globals.outputs.VALIDATOR_SCHEDULE_DESCRIPTION }}" \ + --schedule="${{ steps.globals.outputs.VALIDATOR_SCHEDULE_CRON }}" \ + --time-zone=America/Denver \ + --location=us-central1 \ + --topic="${{ steps.globals.outputs.TOPIC_NAME }}" \ + --message-body='validate' \ + --quiet + else + gcloud scheduler jobs update pubsub "${{ steps.globals.outputs.VALIDATOR_SCHEDULE_NAME }}" \ + --description="${{ steps.globals.outputs.VALIDATOR_SCHEDULE_DESCRIPTION }}" \ + --schedule="${{ steps.globals.outputs.VALIDATOR_SCHEDULE_CRON }}" \ + --time-zone=America/Denver \ + --location=us-central1 \ + --topic="${{ steps.globals.outputs.TOPIC_NAME }}" \ + --message-body='validate' \ + --quiet + fi diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml new file mode 100644 index 0000000..d2324e9 --- /dev/null +++ b/.github/workflows/pull_request.yml @@ -0,0 +1,39 @@ +name: Pull Request Events + +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + test-unit: + name: Unit Tests + runs-on: ubuntu-latest + + steps: + - name: โฌ‡๏ธ Set up code + uses: actions/checkout@v4 + with: + show-progress: false + + - name: ๐Ÿ Set up Python + uses: actions/setup-python@v5 + with: + cache: pip + cache-dependency-path: setup.py + + - name: ๐Ÿ“ฅ Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y libkrb5-dev + + - name: ๐Ÿ— Install module + run: pip install .[tests] + + - name: ๐Ÿงถ Lint + run: ruff check --output-format=github . + + - name: ๐Ÿงช Run pytest + run: pytest diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 183b4d4..92dbb49 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -2,59 +2,38 @@ name: Push Events on: push: - branches: - - main - - dev - pull_request: - branches: - - main - - dev - -env: - CLOUD_FUNCTION_MEMORY: 512M - CLOUD_FUCNTION_RUN_TIMEOUT: 240s - SCHEDULE_NAME: monday-morning - SCHEDULE_CRON: 0 9 * * 1 - SCHEDULE_DESCRIPTION: "Trigger the projectname-skid bot once a week on monday morning" concurrency: - group: "${{ github.head_ref || github.ref }}" + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: - test: - name: Setup and Test + release-please: + name: Create release + if: github.ref_name == 'main' runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - show-progress: false - - - name: Set up Python - uses: actions/setup-python@v5 + - name: ๐Ÿš€ Create Release + id: release-please + uses: agrc/release-composite-action@v1 with: - python-version: 3.11 - cache: pip - cache-dependency-path: setup.py - - - name: Install libkrb5 for Kerberos on Linux - run: | - sudo apt-get update - sudo apt-get install -y libkrb5-dev - - - name: Install module - run: pip install .[tests] - - - name: Test with pytest - run: pytest + release-type: python + prerelease: ${{ github.ref_name == 'dev' }} + repo-token: ${{ secrets.GITHUB_TOKEN }} + github-app-id: ${{ secrets.UGRC_RELEASE_BOT_APP_ID }} + github-app-key: ${{ secrets.UGRC_RELEASE_BOT_APP_KEY }} + github-app-name: ${{ secrets.UGRC_RELEASE_BOT_NAME }} + github-app-email: ${{ secrets.UGRC_RELEASE_BOT_EMAIL }} + extra-files: src/skidname/version.py deploy-dev: - name: Deploy to GCF - needs: test + name: Deploy to GCF - dev runs-on: ubuntu-latest - if: github.ref == 'refs/heads/dev' + if: github.ref_name == 'dev' environment: name: dev permissions: @@ -64,117 +43,14 @@ jobs: steps: - name: โฌ‡๏ธ Set up code uses: actions/checkout@v4 - - - name: ๐Ÿ—๏ธ Authenticate to Google Cloud - id: auth - uses: google-github-actions/auth@v2 - with: - create_credentials_file: true - token_format: access_token - workload_identity_provider: ${{ secrets.IDENTITY_PROVIDER }} - service_account: ${{ secrets.SERVICE_ACCOUNT_EMAIL }} - - - name: ๐Ÿš€ Deploy to Cloud Function - id: deploy - uses: google-github-actions/deploy-cloud-functions@v3 - timeout-minutes: 15 - with: - name: projectname-skid - runtime: python311 - entry_point: subscribe - source_dir: src/projectname - service_account: cloud-function-sa@${{ secrets.PROJECT_ID }}.iam.gserviceaccount.com - event_trigger_type: google.cloud.pubsub.topic.v1.messagePublished - event_trigger_pubsub_topic: projects/${{ secrets.PROJECT_ID }}/topics/${{ env.SCHEDULE_NAME }}-topic - memory: ${{ env.CLOUD_FUNCTION_MEMORY }} - service_timeout: ${{ env.CLOUD_FUNCTION_RUN_TIMEOUT }} - environment_variables: STORAGE_BUCKET=${{secrets.STORAGE_BUCKET}} - secrets: | - /secrets/app/secrets.json=${{secrets.PROJECT_ID}}/skid-secrets - max_instance_count: 1 - event_trigger_retry: false - - - name: ๐Ÿ“ฅ Create PubSub topic - run: | - if [ ! "$(gcloud pubsub topics list | grep $SCHEDULE_NAME-topic)" ]; then - gcloud pubsub topics create $SCHEDULE_NAME-topic --quiet - fi - - - name: ๐Ÿ•ฐ๏ธ Create Cloud Scheduler - run: | - for i in $(gcloud scheduler jobs list --location=us-central1 --uri); do - gcloud scheduler jobs delete $i --quiet - done - gcloud scheduler jobs create pubsub $SCHEDULE_NAME \ - --description="$SCHEDULE_DESCRIPTION" \ - --schedule="$SCHEDULE_CRON" \ - --time-zone=America/Denver \ - --location=us-central1 \ - --topic=$SCHEDULE_NAME-topic \ - --message-body='foo' \ - --quiet - - deploy-prod: - name: Deploy to GCF - needs: test - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' - environment: - name: prod - permissions: - id-token: write - contents: read - - steps: - - name: โฌ‡๏ธ Set up code - uses: actions/checkout@v4 - - - name: ๐Ÿ—๏ธ Authenticate to Google Cloud - id: auth - uses: google-github-actions/auth@v2 with: - create_credentials_file: true - token_format: access_token - workload_identity_provider: ${{ secrets.IDENTITY_PROVIDER }} - service_account: ${{ secrets.SERVICE_ACCOUNT_EMAIL }} + show-progress: false - - name: ๐Ÿš€ Deploy to Cloud Function - id: deploy - uses: google-github-actions/deploy-cloud-functions@v3 + - name: Deploy + uses: ./.github/actions/deploy timeout-minutes: 15 with: - name: projectname-skid - runtime: python311 - entry_point: main - source_dir: src/projectname - service_account: cloud-function-sa@${{ secrets.PROJECT_ID }}.iam.gserviceaccount.com - event_trigger_type: google.cloud.pubsub.topic.v1.messagePublished - event_trigger_pubsub_topic: projects/${{ secrets.PROJECT_ID }}/topics/${{ env.SCHEDULE_NAME }}-topic - memory: ${{ env.CLOUD_FUNCTION_MEMORY }} - service_timeout: ${{ env.CLOUD_FUNCTION_RUN_TIMEOUT }} - environment_variables: STORAGE_BUCKET=${{secrets.STORAGE_BUCKET}} - secrets: | - /secrets/ftp/known_hosts=${{secrets.PROJECT_ID}}/known_hosts - /secrets/app/secrets.json=${{secrets.PROJECT_ID}}/skid-secrets - max_instance_count: 1 - event_trigger_retry: false - - - name: ๐Ÿ“ฅ Create PubSub topic - run: | - if [ ! "$(gcloud pubsub topics list | grep $SCHEDULE_NAME-topic)" ]; then - gcloud pubsub topics create $SCHEDULE_NAME-topic --quiet - fi - - - name: ๐Ÿ•ฐ๏ธ Create Cloud Scheduler - run: | - for i in $(gcloud scheduler jobs list --location=us-central1 --uri); do - gcloud scheduler jobs delete $i --quiet - done - gcloud scheduler jobs create pubsub $SCHEDULE_NAME \ - --description="$SCHEDULE_DESCRIPTION" \ - --schedule="$SCHEDULE_CRON" \ - --time-zone=America/Denver \ - --location=us-central1 \ - --topic=$SCHEDULE_NAME-topic \ - --message-body='{"run": "now"}' \ - --quiet + project_id: ${{ secrets.PROJECT_ID }} + identity_provider: ${{ secrets.IDENTITY_PROVIDER }} + service_account_email: ${{ secrets.SERVICE_ACCOUNT_EMAIL }} + storage_bucket: ${{ secrets.STORAGE_BUCKET }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..959b25d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,53 @@ +name: Release Events +on: + release: + types: [published] + +jobs: + deploy-prod: + name: Deploy to GCF - prod + runs-on: ubuntu-latest + environment: + name: prod + permissions: + id-token: write + contents: read + + steps: + - name: โฌ‡๏ธ Set up code + uses: actions/checkout@v4 + with: + show-progress: false + + - name: Deploy + uses: ./.github/actions/deploy + timeout-minutes: 15 + with: + project_id: ${{ secrets.PROJECT_ID }} + identity_provider: ${{ secrets.IDENTITY_PROVIDER }} + service_account_email: ${{ secrets.SERVICE_ACCOUNT_EMAIL }} + storage_bucket: ${{ secrets.STORAGE_BUCKET }} + + - name: ๐Ÿ”” Create deployment notification + uses: agrc/service-now-worknote-action@v1 + with: + repo-token: ${{ github.token }} + username: ${{ secrets.SN_USERNAME }} + password: ${{ secrets.SN_PASSWORD }} + instance-name: ${{ secrets.SN_INSTANCE }} + table-name: ${{ secrets.SN_TABLE }} + system-id: ${{ secrets.SN_SYS_ID }} + + notify: + name: Notifications + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + issues: write + + steps: + - name: Release Notifier + uses: agrc/release-issue-notifications-action@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index 40c4ea5..1adbc7f 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ If you make changes to your code, you need to kill (ctrl-c) and restart function Skids run as Cloud Functions triggered by Cloud Scheduler sending a notification to a pub/sub topic on a regular schedule. -Work with the GCP maestros to set up a Google project via terraform. They can use the erap configuration as a starting point. Skids use some or all of the following GCP resources: +Work with the GCP maestros to set up a Google project via terraform. They can use the wmrc configuration as a starting point. Skids use some or all of the following GCP resources: - Cloud Functions (executes the python) - Cloud Storage (writing the data files and log files for mid-term retention) @@ -90,14 +90,29 @@ Work with the GCP maestros to set up a Google project via terraform. They can us ### Setup GitHub CI Pipeline -Skids use a GitHub action to deploy the function, pub/sub topic, and scheduler action to the GCP project. They use the following GitHub secrets to do this: +Skids use a series of GitHub actions to test, release, and deploy the skid, including deploying to both dev and prod GCP environments. + +The GCP deployment itself is found within the `deploy` composite action (`.github/actions/deploy/action.yml`). You will set the various GCP parameters (schedule, name, etc) in either the global variables in the `Set globals` step or directly in the main `Deploy Cloud Function` step. + +The cloud functions may need 512 MB or 1 GB of RAM to run successfully. The source dir should point to `src/skidname`. A cloud function just runs the specified function in the `main.py` file in the source dir; it doesn't pip install the function itself. It will pip install any dependencies listed in `src/skidname/requirements.txt`, however. + +The standard workflow, it goeth thusly: + +- Any pushes to the `dev` branch will deploy the skid to the dev GCP environment so that you can test your work in a real-world environment. +- Opening a pull request on any branch triggers the tests to run. +- Pushes to main (usually from merging a PR into main, but also direct pushes) will open a release PR to create a GitHub release with an associated git tag and version bump if the commits pushed contain conventional commit types that trigger a release. +- Once the GitHub release is created by merging the release PR, the actions will deploy the skid to the prod GCP environment. + +The actions rely on several GitHub secrets to do all this: - Identity provider - GCP service account email - Project ID - Storage Bucket ID -The cloud functions may need 512 MB or 1 GB of RAM to run successfully. The source dir should point to `src/skidname`. A cloud function just runs the specified function in the `main.py` file in the source dir; it doesn't pip install the function itself. It will pip install any dependencies listed in `setup.py`, however. +### Multiple Triggers with Different Schedules + +You can parameterize your function to perform different actions depending on the schedule that calls it. The `message-body` parameter of the PubSub topic schedule is available to the `subscribe` function in `main.py`. Therefore, you can create multiple schedules with different `message-body` values and switch the skid functionality based on the values. One example is to have a main routine that executes on a regular basis, and a utility routine that runs a couple times a year. ### Handling Secrets and Configuration Files diff --git a/src/skidname/requirements.txt b/src/skidname/requirements.txt new file mode 100644 index 0000000..192ba2f --- /dev/null +++ b/src/skidname/requirements.txt @@ -0,0 +1,3 @@ +ugrc-palletjack==5.0.* +agrc-supervisor==3.0.3 +requests<2.32