Merge branch 'main' into thuang-765-next-js

.github/workflows/push-rdev.yml

@@ -0,0 +1,72 @@
+name: Push Remote Dev
+
+on:
+  push:
+    branches:
+      - 'rdev-*'
+
+env:
+  DEPLOYMENT_STAGE: test
+  # Force using BuildKit instead of normal Docker, required so that metadata
+  # is written/read to allow us to use layers of previous builds as cache.
+  DOCKER_BUILDKIT: 1
+  COMPOSE_DOCKER_CLI_BUILD: 1
+  DOCKER_REPO: ${{ secrets.ECR_REPO }}/
+
+jobs:
+  build-push-images:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        component: ['frontend', 'backend', 'upload_failures', 'processing']
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          role-duration-seconds: 900
+      - name: Login to ECR
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ secrets.ECR_REPO }}
+      - uses: actions/checkout@v2
+      - name: Build component
+        shell: bash
+        run: |
+          pip install -r .happy/requirements.txt
+          pip install -r .happy/requirements2.txt
+          scripts/happy --profile="" push --extra-tag sha-${GITHUB_SHA:0:8} --extra-tag build-${GITHUB_RUN_NUMBER} ${{ matrix.component }}
+
+  create-update-rdev:
+    runs-on: ubuntu-20.04
+    needs:
+      - build-push-images
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          role-duration-seconds: 900
+      - uses: actions/checkout@v2
+      - name: Install happy dependencies
+        run: |
+          pip install -r .happy/requirements.txt
+          pip install -r .happy/requirements2.txt
+      - name: Create update rdev
+        env:
+          TFE_TOKEN: ${{ secrets.TFE_TOKEN }}
+        run: |
+          RDEV_NAME=${GITHUB_REF#refs/heads/rdev-}
+          if $(./scripts/happy --profile="" list | grep -q $RDEV_NAME); then
+            echo "Updating stack $RDEV_NAME"
+            ./scripts/happy --profile=""  update --tag build-${GITHUB_RUN_NUMBER} $RDEV_NAME
+          else
+            echo "Creating stack $RDEV_NAME"
+            ./scripts/happy --profile="" create  --tag build-${GITHUB_RUN_NUMBER} $RDEV_NAME
+          fi

REMOTE_DEV.md

@@ -48,6 +48,9 @@ Options:
   --help              Show this message and exit.
 ```
 
+### GitHub Action Ingegration
+A new stack can also be deployed to remote development environment through GitHub Action integration. Pushing any branch prefixed with "rdev-" will trigger the GH Action workflow to create or update a dev stack, with the stack name equals the part of branch name following the prefix, e.g. pushing branch "rdev-my-dev-branch" will deploy the stack "my-dev-branch" in the remote dev enviroment. This is useful in situation where local connections is slow.
+
 ### Authentication
 The backend of Happy uses CZI's deployment of Terraform Enterprise (TFE) to deploy and track the resources
 for the stacks. This requires logging into TFE to get a long-lived token tied to your user.

scripts/happy-deploy.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+import click
+import dateutil.parser as dp
+import json
+import os
+import requests
+
+github_org = "chanzuckerberg"
+github_repo = "corpora-data-portal"
+github_graphql_endpoint = "https://api.github.com/graphql"
+github_deployment_endpoint = "https://api.github.com/repos/chanzuckerberg/corpora-data-portal/deployments"
+
+
+def get_latest_successfull_deployment(github_api_token, stage):
+    """get the latest successful/active deployment githun sha"""
+    query = """
+    query($repo_owner:String!, $repo_name:String!, $deployment_env:String!) { 
+        repository(owner: $repo_owner, name: $repo_name) { 
+          deployments(environments: [$deployment_env], first: 30) { 
+            nodes { 
+              commitOid 
+              statuses(first: 100) { 
+                nodes { 
+                  state 
+                  updatedAt 
+                } 
+              } 
+            } 
+          } 
+        } 
+      }
+      """
+
+    variables = {"repo_owner": github_org, "repo_name": github_repo, "deployment_env": stage}
+
+    headers = {"Authorization": "token %s" % github_api_token}
+    query = {"query": query, "variables": variables}
+
+    try:
+        resp = requests.post(url=github_graphql_endpoint, json=query, headers=headers)
+        if resp.status_code != 200:
+            print("Error: Unexpected response {}".format(resp))
+            return None
+    except requests.exceptions.RequestException as e:
+        print("Error: {}".format(e))
+        return None
+
+    resp_json = json.loads(resp.text)
+    deployments = resp_json["data"]["repository"]["deployments"]
+
+    sha_tuple = (None, None)
+
+    for node in deployments["nodes"]:
+        gh_sha = node["commitOid"]
+        for status in node["statuses"]["nodes"]:
+            if status["state"] == "SUCCESS":
+                parsed_t = dp.parse(status["updatedAt"])
+                if sha_tuple[0] == None:
+                    sha_tuple = (gh_sha, parsed_t)
+                else:
+                    if sha_tuple[1] < parsed_t:
+                        sha_tuple = (gh_sha, parsed_t)
+            break
+
+    return sha_tuple[0]
+
+
+def trigger_deploy(github_api_token, deployment_stage, github_sha):
+    """Start deployment to the given environment based on the github sha"""
+    headers = {"Authorization": "token %s" % github_api_token, "Accept": "application/vnd.github.v3.text-match+json"}
+
+    tag = f"sha-{github_sha[0:8]}"
+
+    params = {
+        "ref": github_sha,
+        "auto_merge": False,
+        "environment": deployment_stage,
+        "required_contexts": [],
+        "payload": {"tag": tag},
+    }
+
+    print(f"Deploying {tag} to environment {deployment_stage}")
+    try:
+        resp = requests.post(github_deployment_endpoint, headers=headers, json=params)
+        if resp.status_code != 201:
+            print("Error: Unexpected response {}".format(resp))
+            return
+    except requests.exceptions.RequestException as e:
+        print("Error: {}".format(e))
+        return
+
+    print("Deployment successful")
+
+
+@click.command()
+@click.argument("deployment_stage")
+@click.option("--github_sha", help="github sha to be deployed", default=None)
+def happy_deploy(deployment_stage, github_sha):
+    api_token = os.getenv("GITHUB_TOKEN")
+    if api_token is None:
+        print("Error: Please set GITHUB_TOKEN environment variable")
+        return
+
+    # If github sha is not provided, get the latest succesfull deployment
+    # github sha of staging environment
+    if github_sha is None:
+        github_sha = get_latest_successfull_deployment(api_token, "stage")
+
+    # Trigger deployment on the given stage. This will trigger github actions
+    # and start/update the deployment.
+    if github_sha is not None:
+        trigger_deploy(api_token, deployment_stage, github_sha)
+
+
+if __name__ == "__main__":
+    happy_deploy()