Add Prow jobs to automate the release of Docker images and Python SDK

.prow/config.yaml

@@ -2,32 +2,41 @@ prowjob_namespace: default
 pod_namespace: test-pods
 
 plank:
-  allow_cancellations: true
-  job_url_prefix: http://prow.feast.ai/view/gcs
+  job_url_prefix_config:
+    "*": http://prow.feast.ai/view/gcs
   report_template: '[Full PR test history](https://prow.feast.ai/pr-history?org={{.Spec.Refs.Org}}&repo={{.Spec.Refs.Repo}}&pr={{with index .Spec.Refs.Pulls 0}}{{.Number}}{{end}})'
-  default_decoration_config:
-    timeout: 7200000000000    # 2h
-    grace_period: 15000000000 # 15s
-    utility_images:
-      clonerefs: gcr.io/k8s-prow/clonerefs:v20190221-d14461a
-      initupload: gcr.io/k8s-prow/initupload:v20190221-d14461a
-      entrypoint: gcr.io/k8s-prow/entrypoint:v20190221-d14461a
-      sidecar: gcr.io/k8s-prow/sidecar:v20190221-d14461a
-    gcs_configuration:
-      bucket: feast-templocation-kf-feast
-      path_strategy: explicit
-    gcs_credentials_secret: prow-service-account
+  pod_pending_timeout: 60m
+  default_decoration_configs:
+    "*":
+      timeout: 1h
+      grace_period: 15s
+      utility_images:
+        clonerefs: gcr.io/k8s-prow/clonerefs:v20190221-d14461a
+        initupload: gcr.io/k8s-prow/initupload:v20190221-d14461a
+        entrypoint: gcr.io/k8s-prow/entrypoint:v20190221-d14461a
+        sidecar: gcr.io/k8s-prow/sidecar:v20190221-d14461a
+      gcs_configuration:
+        bucket: feast-templocation-kf-feast
+        path_strategy: explicit
+      gcs_credentials_secret: prow-service-account
 
 deck:
   tide_update_period: 1s
   spyglass:
-    size_limit: 50e+6 # 50MB
-    viewers:
-      "started.json|finished.json": ["metadata"]
-      "build-log.txt": ["buildlog"]
-      "report.xml": ["junit"]
-      "artifacts/.*\\.xml": ["junit"]
-      "surefire-reports/.*\\.xml": ["junit"]
+    size_limit: 10e+6 # 10MB
+    lenses:
+    - lens:
+        name: metadata
+      required_files:
+      - started.json|finished.json
+    - lens:
+        name: buildlog
+      required_files:
+      - build-log.txt
+    - lens:
+        name: junit
+      required_files:
+      - artifacts/.*\.xml
 
 tide:
   queries:
@@ -47,7 +56,9 @@ tide:
   blocker_label: merge-blocker
   squash_label: tide/squash
 
-# presubmits list Prow jobs that run on pull requests
+# presubmits and postsubmits configure ProwJobs:
+# https://github.com/kubernetes/test-infra/blob/6571843b1aa7bd6cf577a7a8b9e9971241f424d5/prow/jobs.md
+
 presubmits:
   gojek/feast:
   - name: test-core-and-ingestion
@@ -131,8 +142,140 @@ presubmits:
         - name: service-account
           mountPath: "/etc/service-account"
 
-# TODO: do a release when a git tag is pushed
-#
-# postsubmits list Prow jobs that run on every push 
-# postsubmits:
-#   gojek/feast:
+postsubmits:
+  gojek/feast:
+  - name: publish-python-sdk
+    decorate: true        
+    spec:
+      containers:
+      - image: python:3
+        command:
+        - sh
+        - -c 
+        - |
+          .prow/scripts/publish-python-sdk.sh \
+            --directory-path sdk/python --repository pypi
+        volumeMounts:
+        - name: pypirc 
+          mountPath: /root/.pypirc
+          subPath: .pypirc
+          readOnly: true
+      volumes:
+      - name: pypirc
+        secret:
+          secretName: pypirc
+    branches:
+    # Filter on tags with semantic versioning, prefixed with "v"
+    # https://github.com/semver/semver/issues/232
+    - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$
+
+  - name: publish-docker-images
+    decorate: true 
+    spec:
+      containers:
+      - image: google/cloud-sdk:273.0.0
+        command:
+        - bash
+        - -c
+        - |
+          .prow/scripts/download-maven-cache.sh \
+            --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
+            --output-dir $PWD/
+
+          if [ $PULL_BASE_REF == "master" ]; then 
+
+          .prow/scripts/publish-docker-image.sh \
+            --repository gcr.io/kf-feast/feast-core \
+            --tag dev \
+            --file infra/docker/core/Dockerfile \
+            --google-service-account-file /etc/gcloud/service-account.json
+
+          .prow/scripts/publish-docker-image.sh \
+            --repository gcr.io/kf-feast/feast-serving \
+            --tag dev \
+            --file infra/docker/serving/Dockerfile \
+            --google-service-account-file /etc/gcloud/service-account.json
+
+          docker tag gcr.io/kf-feast/feast-core:dev gcr.io/kf-feast/feast-core:${PULL_BASE_SHA}
+          docker push gcr.io/kf-feast/feast-core:${PULL_BASE_SHA}
+
+          docker tag gcr.io/kf-feast/feast-serving:dev gcr.io/kf-feast/feast-serving:${PULL_BASE_SHA}
+          docker push gcr.io/kf-feast/feast-serving:${PULL_BASE_SHA}
+
+          else
+
+          .prow/scripts/publish-docker-image.sh \
+            --repository gcr.io/kf-feast/feast-core \
+            --tag ${PULL_BASE_REF:1} \
+            --file infra/docker/core/Dockerfile \
+            --google-service-account-file /etc/gcloud/service-account.json
+
+          .prow/scripts/publish-docker-image.sh \
+            --repository gcr.io/kf-feast/feast-serving \
+            --tag ${PULL_BASE_REF:1} \
+            --file infra/docker/serving/Dockerfile \
+            --google-service-account-file /etc/gcloud/service-account.json
+
+          docker tag gcr.io/kf-feast/feast-core:${PULL_BASE_REF:1} gcr.io/kf-feast/feast-core:latest
+          docker push gcr.io/kf-feast/feast-core:latest
+
+          docker tag gcr.io/kf-feast/feast-serving:${PULL_BASE_REF:1} gcr.io/kf-feast/feast-serving:latest
+          docker push gcr.io/kf-feast/feast-serving:latest
+
+          fi
+        volumeMounts:
+        - name: docker-socket
+          mountPath: /var/run/docker.sock
+        - name: service-account
+          mountPath: /etc/gcloud/service-account.json
+          subPath: service-account.json
+          readOnly: true
+        securityContext:
+          privileged: true
+      volumes:
+      - name: docker-socket
+        hostPath:
+          path: /var/run/docker.sock
+      - name: service-account
+        secret:
+          secretName: feast-service-account
+    branches:
+    - ^master$
+    - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$
+
+  - name: publish-helm-chart
+    decorate: true 
+    spec:
+      containers:
+      - image: google/cloud-sdk:273.0.0-slim
+        command:
+        - bash
+        - -c
+        - |
+          gcloud auth activate-service-account --key-file /etc/gcloud/service-account.json
+
+          curl -s https://get.helm.sh/helm-v2.16.1-linux-amd64.tar.gz | tar -C /tmp -xz
+          mv /tmp/linux-amd64/helm /usr/bin/helm
+          helm init --client-only
+
+          sed -i "/version: /c\version: ${PULL_BASE_REF:1}" infra/charts/feast/Chart.yaml
+          sed -i "/  version: /c\  version: ${PULL_BASE_REF:1}" infra/charts/feast/requirements.yaml
+
+          sed -i "/version: /c\version: ${PULL_BASE_REF:1}" infra/charts/feast/charts/feast-core/Chart.yaml
+          sed -i "/  tag: /c\  tag: ${PULL_BASE_REF:1}" infra/charts/feast/charts/feast-core/values.yaml
+
+          sed -i "/version: /c\version: ${PULL_BASE_REF:1}" infra/charts/feast/charts/feast-serving/Chart.yaml
+          sed -i "/  tag: /c\  tag: ${PULL_BASE_REF:1}" infra/charts/feast/charts/feast-serving/values.yaml
+
+          .prow/scripts/sync-helm-charts.sh
+        volumeMounts:
+        - name: service-account
+          mountPath: /etc/gcloud/service-account.json
+          subPath: service-account.json
+          readOnly: true
+      volumes:
+      - name: service-account
+        secret:
+          secretName: feast-service-account
+    branches:
+    - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$

.prow/scripts/download-maven-cache.sh

@@ -26,7 +26,7 @@ if [[ ! ${OUTPUT_DIR}  ]]; then usage; exit 1; fi
 # Install Google Cloud SDK if gsutil command not exists
 if [[ ! $(command -v gsutil) ]]; then 
   CURRENT_DIR=$(dirname "$BASH_SOURCE")
-  . "${CURRENT_DIR}"/install_google_cloud_sdk.sh
+  . "${CURRENT_DIR}"/install-google-cloud-sdk.sh
 fi  
 
 gsutil -q cp ${ARCHIVE_URI} /tmp/.m2.tar

.prow/scripts/install_google_cloud_sdk.sh → .prow/scripts/install-google-cloud-sdk.sh

@@ -3,10 +3,10 @@ set -e
 
 usage()
 {
-    echo "usage: . install_google_cloud_sdk.sh
+    echo "usage: . install-google-cloud-sdk.sh
     [--with-key-file   local file path to service account json]
 
-NOTE: requires 'dot' before install_google_cloud_sdk.sh
+NOTE: requires 'dot' before install-google-cloud-sdk.sh
       so that the PATH variable  is exported succesfully to 
       the calling process, i.e. you don't need to provide
       full path to gcloud command after installation

.prow/scripts/publish-docker-image.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+set -e 
+set -o pipefail
+
+usage()
+{
+    echo "usage: publish-docker-image.sh
+
+    --repository  the target repository to upload the Docker image, example:
+                  gcr.io/kf-feast/feast-core
+
+    --tag         the tag for the Docker image, example: 1.0.4
+
+    --file        path to the Dockerfile
+
+    [--google-service-account-file  
+    path to Google Cloud service account JSON key file]
+"
+}
+
+while [ "$1" != "" ]; do
+  case "$1" in
+      --repository )           REPOSITORY="$2";         shift;;
+      --tag        )           TAG="$2";                shift;;
+      --file       )           FILE="$2";               shift;;
+      --google-service-account-file ) GOOGLE_SERVICE_ACCOUNT_FILE="$2";        shift;;
+      -h | --help )            usage;                   exit;; 
+      * )                      usage;                   exit 1
+  esac
+  shift
+done
+
+if [ -z $REPOSITORY ]; then usage; exit 1; fi
+if [ -z $TAG ]; then usage; exit 1; fi
+if [ -z $FILE ]; then usage; exit 1; fi
+
+if [ $GOOGLE_SERVICE_ACCOUNT_FILE ]; then 
+    gcloud -q auth activate-service-account --key-file $GOOGLE_SERVICE_ACCOUNT_FILE
+    gcloud -q auth configure-docker
+fi
+
+echo "============================================================"
+echo "Building Docker image $REPOSITORY:$TAG"
+echo "============================================================"
+docker build -t $REPOSITORY:$TAG --build-arg REVISION=$TAG -f $FILE .
+
+echo "============================================================"
+echo "Pushing Docker image $REPOSITORY:$TAG"
+echo "============================================================"
+docker push $REPOSITORY:$TAG

.prow/scripts/publish-python-sdk.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+set -e 
+set -o pipefail
+
+usage()
+{
+    echo "usage: publish-python-sdk.sh
+
+    --directory-path  absolute path to the python package, this directory 
+                      should contain 'setup.py' file
+
+    --repository      the repository name where the package will be uploaded,
+                      check your .pypirc configuration file for the list of 
+                      valid repositories, usually it's 'pypi' or 'testpypi'
+"
+}
+
+while [ "$1" != "" ]; do
+  case "$1" in
+      --directory-path )       DIRECTORY_PATH="$2";     shift;;
+      --repository )           REPOSITORY="$2";         shift;;
+      -h | --help )            usage;                   exit;; 
+      * )                      usage;                   exit 1
+  esac
+  shift
+done
+
+if [ -z $DIRECTORY_PATH ]; then usage; exit 1; fi
+if [ -z $REPOSITORY ]; then usage; exit 1; fi
+
+ORIGINAL_DIR=$PWD
+cd $DIRECTORY_PATH
+
+echo "============================================================"
+echo "Generating distribution archives"
+echo "============================================================"
+python3 -m pip install --user --upgrade setuptools wheel
+python3 setup.py sdist bdist_wheel
+
+echo "============================================================"
+echo "Uploading distribution archives"
+echo "============================================================"
+python3 -m pip install --user --upgrade twine
+python3 -m twine upload --repository $REPOSITORY dist/*
+
+cd $ORIGINAL_DIR

.prow/scripts/sync-helm-charts.sh

@@ -46,12 +46,12 @@ if helm repo index --url "$repo_url" --merge "$index_dir/index.yaml" "$sync_dir"
     gsutil -m rsync "$sync_dir" "$bucket"
 
     # Make sure index.yaml is synced last
-    gsutil cp "$index_dir/index.yaml" "$bucket"
+    gsutil -h "Cache-Control:no-cache,max-age=0" cp "$index_dir/index.yaml" "$bucket"
 else
     log_error "Exiting because unable to update index. Not safe to push update."
     exit 1
 fi
 
 ls -l "$sync_dir"
 
-exit "$exit_code"
+exit "$exit_code"

.prow/scripts/test-end-to-end-batch.sh

@@ -28,7 +28,7 @@ Installing gcloud SDK
 "
 if [[ ! $(command -v gsutil) ]]; then
   CURRENT_DIR=$(dirname "$BASH_SOURCE")
-  . "${CURRENT_DIR}"/install_google_cloud_sdk.sh
+  . "${CURRENT_DIR}"/install-google-cloud-sdk.sh
 fi
 
 export GOOGLE_APPLICATION_CREDENTIALS=/etc/service-account/service-account.json
@@ -210,7 +210,7 @@ bash /tmp/miniconda.sh -b -p /root/miniconda -f
 source ~/.bashrc
 
 # Install Feast Python SDK and test requirements
-pip install -q sdk/python
+pip install -qe sdk/python
 pip install -qr tests/e2e/requirements.txt
 
 echo "

.prow/scripts/test-end-to-end.sh

@@ -191,7 +191,7 @@ bash /tmp/miniconda.sh -b -p /root/miniconda -f
 source ~/.bashrc
 
 # Install Feast Python SDK and test requirements
-pip install -q sdk/python
+pip install -qe sdk/python
 pip install -qr tests/e2e/requirements.txt
 
 echo "

.prow/scripts/test-python-sdk.sh

@@ -7,5 +7,5 @@ LOGS_ARTIFACT_PATH=/logs/artifacts
 
 cd sdk/python
 pip install -r requirements-ci.txt
-pip install .
+pip install -e .
 pytest --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml