Update Prow configuration for Feast CI

.prow/config.yaml

@@ -0,0 +1,140 @@
+prowjob_namespace: default
+pod_namespace: test-pods
+
+plank:
+  allow_cancellations: true
+  job_url_prefix: http://prow.feast.ai/view/gcs
+  report_template: '[Full PR test history](https://prow.feast.ai/pr-history?org={{.Spec.Refs.Org}}&repo={{.Spec.Refs.Repo}}&pr={{with index .Spec.Refs.Pulls 0}}{{.Number}}{{end}})'
+  default_decoration_config:
+    timeout: 7200000000000    # 2h
+    grace_period: 15000000000 # 15s
+    utility_images:
+      clonerefs: gcr.io/k8s-prow/clonerefs:v20190221-d14461a
+      initupload: gcr.io/k8s-prow/initupload:v20190221-d14461a
+      entrypoint: gcr.io/k8s-prow/entrypoint:v20190221-d14461a
+      sidecar: gcr.io/k8s-prow/sidecar:v20190221-d14461a
+    gcs_configuration:
+      bucket: feast-templocation-kf-feast
+      path_strategy: explicit
+    gcs_credentials_secret: prow-service-account
+
+deck:
+  tide_update_period: 1s
+  spyglass:
+    size_limit: 100e+6 # 100MB
+    viewers:
+      "started.json|finished.json": ["metadata"]
+      "build-log.txt": ["buildlog"]
+      "report.xml": ["junit"]
+      "artifacts/.*\\.xml": ["junit"]
+      "surefire-reports/.*\\.xml": ["junit"]
+
+tide:
+  queries:
+  - repos:
+    - gojek/feast
+    labels:
+    - lgtm
+    - approved
+    missingLabels:
+    - do-not-merge
+    - do-not-merge/hold
+    - do-not-merge/invalid-owners-file
+    - do-not-merge/work-in-progress
+    - needs-rebase
+  merge_method:
+    gojek/feast: squash
+  blocker_label: merge-blocker
+  squash_label: tide/squash
+
+# presubmits list Prow jobs that run on pull requests
+presubmits:
+  gojek/feast:
+  - name: unit-test-core
+    decorate: true
+    always_run: true
+    spec:
+      containers:
+      - image: maven:3.6-jdk-8
+        command: [".prow/scripts/run_unit_test.sh", "--component", "core"]
+
+  - name: unit-test-ingestion
+    decorate: true
+    always_run: true
+    spec:
+      containers:
+      - image: maven:3.6-jdk-8
+        command: [".prow/scripts/run_unit_test.sh", "--component", "ingestion"]
+
+  - name: unit-test-serving
+    decorate: true
+    always_run: true
+    spec:
+      containers:
+      - image: maven:3.6-jdk-8
+        command: [".prow/scripts/run_unit_test.sh", "--component", "serving"]
+
+  - name: unit-test-cli
+    decorate: true
+    always_run: true
+    spec:
+      containers:
+      - image: golang:1.12
+        env:
+        - name: GO111MODULE
+          value: "on"
+        command: [".prow/scripts/run_unit_test.sh", "--component", "cli"]
+
+  - name: integration-test
+    decorate: true
+    always_run: true
+    spec:
+      volumes:
+      - name: docker-socket-volume
+        hostPath:
+          path: /var/run/docker.sock
+          type: File
+      - name: service-account
+        secret:
+          secretName: prow-service-account
+      nodeSelector:
+        os: ubuntu
+      containers:
+      - image: google/cloud-sdk
+        # securityContext and docker socket vol mounts are needed because we are building
+        # Docker images in this job
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: docker-socket-volume
+          mountPath: /var/run/docker.sock
+        - name: service-account
+          mountPath: /etc/service-account
+          readOnly: true
+        command:
+        - bash
+        - -c
+        - |
+          export FEAST_HOME=${PWD}
+          export FEAST_IMAGE_REGISTRY=us.gcr.io
+          export FEAST_IMAGE_TAG=${PULL_PULL_SHA}
+          export FEAST_WAREHOUSE_DATASET=feast_build_${BUILD_ID:0:5}
+          export FEAST_CORE_URL=build-${BUILD_ID:0:5}.drone.feast.ai:80
+          export FEAST_SERVING_URL=build-${BUILD_ID:0:5}.drone.feast.ai:80
+          export FEAST_RELEASE_NAME=feast-${BUILD_ID:0:5}
+          export BATCH_IMPORT_DATA_GCS_PATH=gs://feast-templocation-kf-feast/build_${BUILD_ID:0:5}/integration-tests/testdata/feature_values/ingestion_1.csv
+          export KAFKA_BROKERS=10.128.0.201:9092
+          export KAFKA_TOPICS=feast_build_${BUILD_ID:0:5}
+
+          . .prow/scripts/prepare_integration_test.sh
+          .prow/scripts/install_feast_and_run_e2e_test.sh
+          TEST_EXIT_CODE=$?
+          .prow/scripts/cleanup_feast_installation.sh
+
+          exit ${TEST_EXIT_CODE}
+
+# TODO: do a release when a git tag is pushed
+# postsubmits list Prow jobs that run on every push 
+#
+# postsubmits:
+#   gojek/feast:

.prow/plugins.yaml

@@ -0,0 +1,25 @@
+plugins:
+  gojek/feast:
+  - approve      
+  - assign
+  - help
+  - hold
+  - label
+  - lgtm
+  - lifecycle     
+  - size
+  - verify-owners 
+  - wip         
+  - trigger
+  - config-updater
+
+config_updater:
+  maps:
+    .prow/config.yaml:
+      name: config
+
+external_plugins:
+  gojek/feast:
+  - name: needs-rebase
+    events:
+    - pull_request

.prow/scripts/cleanup_feast_installation.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -e
+
+bq -q rm -rf --dataset ${FEAST_WAREHOUSE_DATASET}
+gsutil -q rm ${BATCH_IMPORT_DATA_GCS_PATH}
+helm delete --purge $FEAST_RELEASE_NAME

.prow/scripts/install_feast_and_run_e2e_test.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo "============================================================"
+echo "Installing Feast Release"
+echo "============================================================"
+
+helm install --name ${FEAST_RELEASE_NAME} --wait --timeout 210 ${FEAST_HOME}/charts/feast -f integration-tests/feast-helm-values.yaml
+
+echo "============================================================"
+echo "Testing Batch Import"
+echo "============================================================"
+
+cd ${FEAST_HOME}/integration-tests/testdata
+
+feast apply entity entity_specs/entity_1.yaml
+feast apply feature feature_specs/entity_1*.yaml
+feast jobs run import_specs/batch_from_gcs.yaml --wait
+
+cd $FEAST_HOME/integration-tests
+
+python -m testutils.validate_feature_values \
+  --entity_spec_file=testdata/entity_specs/entity_1.yaml \
+  --feature_spec_files=testdata/feature_specs/entity_1*.yaml \
+  --expected-warehouse-values-file=testdata/feature_values/ingestion_1.csv \
+  --expected-serving-values-file=testdata/feature_values/serving_1.csv \
+  --bigquery-dataset-for-warehouse=${FEAST_WAREHOUSE_DATASET} \
+  --feast-serving-url=${FEAST_SERVING_URL}
+
+echo "============================================================"
+echo "Testing Streaming Import"
+echo "============================================================"
+
+cd $FEAST_HOME/integration-tests/testdata
+
+feast apply entity entity_specs/entity_2.yaml
+feast apply feature feature_specs/entity_2*.yaml
+feast jobs run import_specs/stream_from_kafka.yaml &
+
+IMPORT_JOB_PID=$!
+sleep 20
+
+cd $FEAST_HOME/integration-tests
+
+python -m testutils.kafka_producer \
+  --bootstrap_servers=$KAFKA_BROKERS \
+  --topic=$KAFKA_TOPICS \
+  --entity_spec_file=testdata/entity_specs/entity_2.yaml \
+  --feature_spec_files=testdata/feature_specs/entity_2*.yaml \
+  --feature_values_file=testdata/feature_values/ingestion_2.csv
+sleep 20
+
+python -m testutils.validate_feature_values \
+  --entity_spec_file=testdata/entity_specs/entity_2.yaml \
+  --feature_spec_files=testdata/feature_specs/entity_2*.yaml \
+  --expected-serving-values-file=testdata/feature_values/serving_2.csv \
+  --feast-serving-url=$FEAST_SERVING_URL
+
+kill -9 ${IMPORT_JOB_PID}

.prow/scripts/install_feast_sdk.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -e
+
+# This script ensures latest Feast Python SDK and Feast CLI are installed
+
+pip install -qe ${FEAST_HOME}/sdk/python
+pip install -qr ${FEAST_HOME}/integration-tests/testutils/requirements.txt
+go build -o /usr/local/bin/feast ./cli/feast &> /dev/null
+feast config set coreURI ${FEAST_CORE_URL}

.prow/scripts/install_google_cloud_sdk.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -e
+
+usage()
+{
+    echo "usage: . install_google_cloud_sdk.sh
+    [--with-key-file   local file path to service account json]
+
+NOTE: requires 'dot' before install_google_cloud_sdk.sh
+      so that the PATH variable  is exported succesfully to 
+      the calling process, i.e. you don't need to provide
+      full path to gcloud command after installation
+
+      --with-key-file is optional, 
+        if no authentication is required"
+}
+
+while [ "$1" != "" ]; do
+  case "$1" in
+      --with-key-file )     KEY_FILE="$2";    shift;;
+      * )                   usage; exit 1
+  esac
+  shift
+done
+
+GOOGLE_CLOUD_SDK_ARCHIVE_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-244.0.0-linux-x86_64.tar.gz
+GOOGLE_PROJECT_ID=kf-feast
+KUBE_CLUSTER_NAME=primary-test-cluster
+KUBE_CLUSTER_ZONE=us-central1-a
+
+curl -s ${GOOGLE_CLOUD_SDK_ARCHIVE_URL} | tar xz -C /
+export PATH=/google-cloud-sdk/bin:${PATH}
+gcloud -q components install kubectl
+
+if [[ ${KEY_FILE} ]]; then 
+    gcloud -q auth activate-service-account --key-file=${KEY_FILE}
+    gcloud -q auth configure-docker
+    gcloud -q config set project ${GOOGLE_PROJECT_ID}
+    gcloud -q container clusters get-credentials ${KUBE_CLUSTER_NAME} --zone ${KUBE_CLUSTER_ZONE} --project ${GOOGLE_PROJECT_ID}
+    export GOOGLE_APPLICATION_CREDENTIALS=${KEY_FILE}
+fi

.prow/scripts/install_test_tools.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -e
+
+# This script installs the following Feast test utilities:
+# ============================================================
+# - gettext package so we can use envsubst command to provide values to helm template file
+# - Python 3.6 because Feast requires Python version 3.6 and above
+# - Golang if we need to build Feast CLI from source
+# - Helm if we want to install Feast release
+
+apt-get -qq update 
+apt-get -y install curl wget gettext &> /dev/null
+
+curl -s https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh -o /tmp/miniconda.sh
+bash /tmp/miniconda.sh -b -p /miniconda &> /dev/null
+export PATH=/miniconda/bin:$PATH
+
+wget -qO- https://dl.google.com/go/go1.12.5.linux-amd64.tar.gz | tar xzf -
+mv go /usr/local/
+export PATH=/usr/local/go/bin:$PATH
+export GO111MODULE=on
+
+wget -qO- https://storage.googleapis.com/kubernetes-helm/helm-v2.13.1-linux-amd64.tar.gz | tar xz
+mv linux-amd64/helm /usr/local/bin/helm

.prow/scripts/prepare_integration_test.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+set -e
+
+usage()
+{
+    echo "usage: prepare_integration_test.sh [--skip-build true]"
+}
+
+while [ "$1" != "" ]; do
+  case "$1" in
+      --skip-build )       SKIP_BUILD=true;              shift;;
+      * )                  usage; exit 1
+  esac
+  shift
+done
+
+# Authenticate to Google Cloud and GKE
+# ============================================================
+GOOGLE_PROJECT_ID=kf-feast
+KUBE_CLUSTER_NAME=primary-test-cluster
+KUBE_CLUSTER_ZONE=us-central1-a
+KEY_FILE=/etc/service-account/service-account.json
+
+gcloud -q auth activate-service-account --key-file=${KEY_FILE}
+gcloud -q auth configure-docker
+gcloud -q config set project ${GOOGLE_PROJECT_ID}
+gcloud -q container clusters get-credentials ${KUBE_CLUSTER_NAME} --zone ${KUBE_CLUSTER_ZONE} --project ${GOOGLE_PROJECT_ID}
+export GOOGLE_APPLICATION_CREDENTIALS=${KEY_FILE}
+
+# Install Python 3.6, Golang 1.12, Helm and Feast SDK
+# ============================================================
+. .prow/scripts/install_test_tools.sh
+. .prow/scripts/install_feast_sdk.sh
+.prow/scripts/prepare_maven_cache.sh --archive-uri gs://feast-templocation-kf-feast/.m2.tar --output-dir ${FEAST_HOME}
+
+# Prepare Feast test data and config
+# ============================================================
+
+bq -q mk --dataset ${FEAST_WAREHOUSE_DATASET}
+gsutil -q cp ${FEAST_HOME}/integration-tests/testdata/feature_values/ingestion_1.csv ${BATCH_IMPORT_DATA_GCS_PATH}
+
+BUILD_ID=${BUILD_ID:0:5}
+envsubst < integration-tests/feast-helm-values.yaml.template > integration-tests/feast-helm-values.yaml
+cd ${FEAST_HOME}/integration-tests/testdata/import_specs
+envsubst < batch_from_gcs.yaml.template > batch_from_gcs.yaml
+envsubst < stream_from_kafka.yaml.template > stream_from_kafka.yaml
+
+if [[ ! ${SKIP_BUILD} ]]; then
+
+echo "============================================================"
+echo "Building Feast for Testing"
+echo "============================================================"
+cd ${FEAST_HOME}
+docker build -t us.gcr.io/kf-feast/feast-core:${FEAST_IMAGE_TAG} -f Dockerfiles/core/Dockerfile . &
+docker build -t us.gcr.io/kf-feast/feast-serving:${FEAST_IMAGE_TAG} -f Dockerfiles/serving/Dockerfile . &
+wait
+docker push us.gcr.io/kf-feast/feast-core:${FEAST_IMAGE_TAG} &
+docker push us.gcr.io/kf-feast/feast-serving:${FEAST_IMAGE_TAG} &
+wait
+
+fi
+
+# Switch back context to original directory
+set +ex
+cd ${FEAST_HOME}