Merge pull request #1 from groovenauts/features/github-actions-releas…

…e-workflow

Automated release workflow: stage application code to cloud storage by tag push, and Go Module

.gcloudignore

@@ -1,10 +1,11 @@
 .git/
+.github/
 .gitignore
 *~
 *.back
 .*.swp
 *.yaml
-pkg/
-makepkg.sh
+runtime
+stage.sh
 README.md
 *.rb

.github/workflows/release.yml

@@ -0,0 +1,82 @@
+#
+# https://help.github.com/en/actions/automating-your-workflow-with-github-actions
+#
+
+name: Release
+
+on:
+  push:
+    tags:
+      - v*
+
+jobs:
+  dump:
+    name: Dump env vars, contexts
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.head_commit.message, '[ci skip]')"
+    steps:
+      - name: Environment Variables
+        run: export -p
+
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - name: Dump job context
+        env:
+          JOB_CONTEXT: ${{ toJson(job) }}
+        run: echo "$JOB_CONTEXT"
+      - name: Dump steps context
+        env:
+          STEPS_CONTEXT: ${{ toJson(steps) }}
+        run: echo "$STEPS_CONTEXT"
+      - name: Dump runner context
+        env:
+          RUNNER_CONTEXT: ${{ toJson(runner) }}
+        run: echo "$RUNNER_CONTEXT"
+      - name: Dump strategy context
+        env:
+          STRATEGY_CONTEXT: ${{ toJson(strategy) }}
+        run: echo "$STRATEGY_CONTEXT"
+      - name: Dump matrix context
+        env:
+          MATRIX_CONTEXT: ${{ toJson(matrix) }}
+        run: echo "$MATRIX_CONTEXT"
+
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    env:
+      GO111MODULE: on
+    steps:
+      # https://github.com/actions/checkout
+      - uses: actions/checkout@v2
+
+      # https://github.com/GoogleCloudPlatform/github-actions/tree/master/setup-gcloud
+      - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+        with:
+          service_account_email: ${{ secrets.GCP_SA_EMAIL }}
+          service_account_key: ${{ secrets.GCP_SA_KEY }}
+
+      - name: Check Cloud SDK version/components
+        run: |
+          set -x
+          gcloud version
+          gcloud info
+          gcloud components list
+          gcloud auth list
+
+      - name: Update and install Cloud SDK components
+        run: |
+          set -x
+          gcloud components update
+          gcloud components install app-engine-go
+          gcloud components list
+
+      - name: Stage application code to Cloud Storage
+        run: |
+          set -x
+          bucket=${{ secrets.GCP_STAGE_BUCKET }}
+          version=$(basename ${{ github.ref }})
+          ./stage.sh ${bucket} ${version}
+

.gitignore

@@ -1,3 +1 @@
 .*.swp
-/pkg/*
-/src

README.md

@@ -3,15 +3,18 @@
 ## Run local server
 
 ```
-goapp serve
+export GO111MODULE=on
+go run magellan-gcs-uploader.go
 ```
 
 ## Deploy
 
 Specify GCP project id and api tokens (comma separated).
 
 ```
-appcfg.py -A YOUR-PROJECT-ID -E STORAGE_BUCKET:YOUR-BUCKET-NAME -E API_TOKEN:XXXXXXXX update .
+export GO111MODULE=on
+(Edit app.yaml to setup environment variables)
+gcloud --project=YOUR-PROJECT-ID app deploy app.yaml
 ```
 
 ### Environment Variables
@@ -28,14 +31,14 @@ appcfg.py -A YOUR-PROJECT-ID -E STORAGE_BUCKET:YOUR-BUCKET-NAME -E API_TOKEN:XXX
 
 ## Upload package to deploy via Google App Engine Admin API
 
-Run the following command To gather source files and make manifest file. `v1` stands for the version of application.
+Run the following command to gather source files and make manifest file, and upload them to gcs.
+`v1` stands for the version of application.
 
 ```
-./makepkg.sh v1
+export GO111MODULE=on
+./makepkg.sh your-gae-repository v1
 ```
 
-Upload source files and a manifest file to gcs.
+This workflow is automated by [Release workflow](.github/workflows/release.yml),
+triggered by tag push.
 
-```
-gsutil cp -R pkg/v1 gs://your-gae-repository/magellan-gcs-uploader/v1
-```

go.mod

@@ -0,0 +1,7 @@
+module github.com/groovenauts/magellan-gcs-uploader
+
+require (
+	cloud.google.com/go/bigquery v1.4.0
+	cloud.google.com/go/storage v1.5.0
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
+)

runtime

@@ -0,0 +1 @@
+go111

stage.sh

@@ -0,0 +1,83 @@
+#! /bin/bash
+
+set -eo pipefail
+
+app_name=$(basename -s .git $(git config --get remote.origin.url))
+app_root=$(git rev-parse --show-toplevel)
+project_id=$(gcloud config get-value project)
+
+bucket="$1"
+version="$2"
+
+if [ -z "${bucket}" -o -z "${version}" ]; then
+  echo "Usage: $0 bucket version"
+  exit 1
+fi
+
+gcs_version_path=gs://${bucket}/${app_name}/${version}
+
+echo ">>> app_name: ${app_name}"
+echo ">>> app_root: ${app_root}"
+echo ">>> project_id: ${project_id}"
+echo ">>> bucket: ${bucket}"
+echo ">>> version: ${version}"
+echo ">>> gcs_version_path: ${gcs_version_path}"
+
+# check if already uploaded
+if gsutil -q stat ${gcs_version_path}/_manifest; then
+  echo "Abort. Manifest file ${gcs_version_path}/_manifest is already uploaded."
+  exit 1
+fi
+
+# create temporary directory and clean up on exit
+tmpdir=$(mktemp -d)
+stagedir=$tmpdir/stage
+mkdir $stagedir
+trap "echo '>>> Cleanup'; rm -rfv ${tmpdir}" EXIT
+echo ">>> tmpdir: ${tmpdir}"
+echo ">>> stagedir: ${stagedir}"
+
+# create app.yaml in temporary directory
+runtime=$(cat runtime)
+echo "runtime: ${runtime}" > ${tmpdir}/app.yaml
+echo ">>> runtime: ${runtime}"
+
+# "goNNN" -> "N.NN"
+go_version=${runtime#go}
+go_version=${go_version::1}.${go_version:1}
+echo ">>> go_version: ${go_version}"
+
+# check if go-app-stager is installed
+go_app_stager=$(gcloud info --format='value(installation.sdk_root)')/platform/google_appengine/go-app-stager
+if [ ! -x "${go_app_stager}" ]; then
+  echo "go-app-stager not found. Run \"gcloud components install app-engine-go\" to install go-app-stager."
+  exit 1
+fi
+
+# copy files into stage directory using go-app-stager
+echo ">>> Copy files by ${go_app_stager} into stagedir"
+${go_app_stager} -go-version=${go_version} ${tmpdir}/app.yaml ${app_root} ${stagedir}
+
+cd ${stagedir}
+
+# upload files to Cloud Storage
+echo ">>> Upload files to cloud storage"
+gcloud meta list-files-for-upload | sort > ${tmpdir}/files-for-upload
+cat ${tmpdir}/files-for-upload | while read f; do
+  echo ">>> * ${PWD}/${f} -> ${gcs_version_path}/${f}"
+  gsutil -q cp ${f} ${gcs_version_path}/${f}
+done
+
+# upload manifest
+echo ">>> Generate and upload menifest file to cloud storage"
+sha1sum $(cat ${tmpdir}/files-for-upload) > ${tmpdir}/_manifest
+echo ">>> * ${tmpdir}/_manifest -> ${gcs_version_path}/_manifest"
+gsutil -q cp ${tmpdir}/_manifest ${gcs_version_path}/_manifest
+
+# list uploaded objects
+echo ">>> List objects in ${gcs_version_path}/"
+gsutil ls -l -r ${gcs_version_path}/
+
+# show manifest
+echo ">>> Show manifest content in ${gcs_version_path}/_manifest"
+gsutil cat ${gcs_version_path}/_manifest