From dfa01e359ad45cb25793e70e19b2ca84bde440e8 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Mon, 9 Dec 2019 18:38:09 +0800 Subject: [PATCH 1/2] Add batch end-to-end test --- .prow/scripts/test-end-to-end-batch.sh | 240 +++++++++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100755 .prow/scripts/test-end-to-end-batch.sh diff --git a/.prow/scripts/test-end-to-end-batch.sh b/.prow/scripts/test-end-to-end-batch.sh new file mode 100755 index 0000000000..3f8fa0db3e --- /dev/null +++ b/.prow/scripts/test-end-to-end-batch.sh @@ -0,0 +1,240 @@ +#!/usr/bin/env bash + +set -e +set -o pipefail + +if ! cat /etc/*release | grep -q stretch; then + echo ${BASH_SOURCE} only supports Debian stretch. + echo Please change your operating system to use this script. + exit 1 +fi + +echo " +This script will run end-to-end tests for Feast Core and Batch Serving. + +1. Install gcloud SDK +2. Install Redis as the job store for Feast Batch Serving. +4. Install Postgres for persisting Feast metadata. +5. Install Kafka and Zookeeper as the Source in Feast. +6. Install Python 3.7.4, Feast Python SDK and run end-to-end tests from + tests/e2e via pytest. +" + + +echo " +============================================================ +Installing gcloud SDK +============================================================ +" +if [[ ! $(command -v gsutil) ]]; then + CURRENT_DIR=$(dirname "$BASH_SOURCE") + . "${CURRENT_DIR}"/install_google_cloud_sdk.sh +fi + +export GOOGLE_APPLICATION_CREDENTIALS=/etc/service-account/service-account.json +gcloud auth activate-service-account --key-file /etc/service-account/service-account.json + + + +echo " +============================================================ +Installing Redis at localhost:6379 +============================================================ +" +apt-get -qq update +# Allow starting serving in this Maven Docker image. Default set to not allowed. +echo "exit 0" > /usr/sbin/policy-rc.d +apt-get -y install redis-server wget > /var/log/redis.install.log +redis-server --daemonize yes +redis-cli ping + +echo " +============================================================ +Installing Postgres at localhost:5432 +============================================================ +" +apt-get -y install postgresql > /var/log/postgresql.install.log +service postgresql start +# Initialize with database: 'postgres', user: 'postgres', password: 'password' +cat < /tmp/update-postgres-role.sh +psql -c "ALTER USER postgres PASSWORD 'password';" +EOF +chmod +x /tmp/update-postgres-role.sh +su -s /bin/bash -c /tmp/update-postgres-role.sh postgres +export PGPASSWORD=password +pg_isready + +echo " +============================================================ +Installing Zookeeper at localhost:2181 +Installing Kafka at localhost:9092 +============================================================ +" +wget -qO- https://www-eu.apache.org/dist/kafka/2.3.0/kafka_2.12-2.3.0.tgz | tar xz +mv kafka_2.12-2.3.0/ /tmp/kafka +nohup /tmp/kafka/bin/zookeeper-server-start.sh /tmp/kafka/config/zookeeper.properties &> /var/log/zookeeper.log 2>&1 & +sleep 5 +tail -n10 /var/log/zookeeper.log +nohup /tmp/kafka/bin/kafka-server-start.sh /tmp/kafka/config/server.properties &> /var/log/kafka.log 2>&1 & +sleep 10 +tail -n10 /var/log/kafka.log + +echo " +============================================================ +Building jars for Feast +============================================================ +" + +.prow/scripts/download-maven-cache.sh \ + --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \ + --output-dir /root/ + +# Build jars for Feast +mvn --quiet --batch-mode --define skipTests=true clean package + +echo " +============================================================ +Starting Feast Core +============================================================ +" +# Start Feast Core in background +cat < /tmp/core.application.yml +grpc: + port: 6565 + enable-reflection: true + +feast: + version: 0.3 + jobs: + runner: DirectRunner + options: {} + metrics: + enabled: false + + stream: + type: kafka + options: + topic: feast-features + bootstrapServers: localhost:9092 + replicationFactor: 1 + partitions: 1 + +spring: + jpa: + properties.hibernate.format_sql: true + hibernate.naming.physical-strategy=org.hibernate.boot.model.naming: PhysicalNamingStrategyStandardImpl + hibernate.ddl-auto: update + datasource: + url: jdbc:postgresql://localhost:5432/postgres + username: postgres + password: password + +management: + metrics: + export: + simple: + enabled: false + statsd: + enabled: false +EOF + +nohup java -jar core/target/feast-core-0.3.2-SNAPSHOT.jar \ + --spring.config.location=file:///tmp/core.application.yml \ + &> /var/log/feast-core.log & +sleep 30 +tail -n10 /var/log/feast-core.log +echo " +============================================================ +Starting Feast Warehouse Serving +============================================================ +" + +DATASET_NAME=feast_$(date +%s) + +bq --location=US --project_id=kf-feast mk \ + --dataset \ + --default_table_expiration 86400 \ + kf-feast:$DATASET_NAME + +# Start Feast Online Serving in background +cat < /tmp/serving.store.bigquery.yml +name: warehouse +type: BIGQUERY +bigquery_config: + projectId: kf-feast + datasetId: $DATASET_NAME +subscriptions: + - name: .* + version: ">0" +EOF + +cat < /tmp/serving.warehouse.application.yml +feast: + version: 0.3 + core-host: localhost + core-grpc-port: 6565 + tracing: + enabled: false + store: + config-path: /tmp/serving.store.bigquery.yml + jobs: + staging-location: gs://feast-templocation-kf-feast/staging-location + store-type: REDIS + store-options: + host: $REMOTE_HOST + port: 6379 +grpc: + port: 6566 + enable-reflection: true +spring: + main: + web-environment: false +EOF + +nohup java -jar serving/target/feast-serving-0.3.0-SNAPSHOT.jar \ + --spring.config.location=file:///tmp/serving.warehouse.application.yml \ + &> /var/log/feast-serving-warehouse.log & +sleep 15 +tail -n10 /var/log/feast-serving-warehouse.log + +echo " +============================================================ +Installing Python 3.7 with Miniconda and Feast SDK +============================================================ +" +# Install Python 3.7 with Miniconda +wget -q https://repo.continuum.io/miniconda/Miniconda3-4.7.12-Linux-x86_64.sh \ + -O /tmp/miniconda.sh +bash /tmp/miniconda.sh -b -p /root/miniconda -f +/root/miniconda/bin/conda init +source ~/.bashrc + +# Install Feast Python SDK and test requirements +pip install -q sdk/python +pip install -qr tests/e2e/requirements.txt + +echo " +============================================================ +Running end-to-end tests with pytest at 'tests/e2e' +============================================================ +" +# Default artifact location setting in Prow jobs +LOGS_ARTIFACT_PATH=/logs/artifacts + +ORIGINAL_DIR=$(pwd) +cd tests/e2e + +set +e +pytest bq-batch-retrieval.py --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml +TEST_EXIT_CODE=$? + +cd ${ORIGINAL_DIR} +exit ${TEST_EXIT_CODE} + +echo " +============================================================ +Cleaning up +============================================================ +" + +bq rm -r -f kf-feast:$DATASET_NAME From fe3a4d12b4b24ab912646b2e98584f1ad89aa5fb Mon Sep 17 00:00:00 2001 From: zhilingc Date: Mon, 9 Dec 2019 20:47:50 +0800 Subject: [PATCH 2/2] Fix issues in the batch tests --- .prow/config.yaml | 21 +++++++++++++++++++++ .prow/scripts/test-end-to-end-batch.sh | 6 +++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.prow/config.yaml b/.prow/config.yaml index fb20159f5a..cf765a9903 100644 --- a/.prow/config.yaml +++ b/.prow/config.yaml @@ -110,6 +110,27 @@ presubmits: limit: memory: "4096Mi" + - name: test-end-to-end-batch + decorate: true + always_run: true + spec: + volumes: + - name: service-account + secret: + secretName: feast-service-account + containers: + - image: maven:3.6-jdk-8 + command: [".prow/scripts/test-end-to-end-batch.sh"] + resources: + requests: + cpu: "1000m" + memory: "1024Mi" + limit: + memory: "4096Mi" + volumeMounts: + - name: service-account + mountPath: "/etc/service-account" + # TODO: do a release when a git tag is pushed # # postsubmits list Prow jobs that run on every push diff --git a/.prow/scripts/test-end-to-end-batch.sh b/.prow/scripts/test-end-to-end-batch.sh index 3f8fa0db3e..82d6286027 100755 --- a/.prow/scripts/test-end-to-end-batch.sh +++ b/.prow/scripts/test-end-to-end-batch.sh @@ -164,7 +164,7 @@ bigquery_config: projectId: kf-feast datasetId: $DATASET_NAME subscriptions: - - name: .* + - name: "*" version: ">0" EOF @@ -191,11 +191,11 @@ spring: web-environment: false EOF -nohup java -jar serving/target/feast-serving-0.3.0-SNAPSHOT.jar \ +nohup java -jar serving/target/feast-serving-0.3.2-SNAPSHOT.jar \ --spring.config.location=file:///tmp/serving.warehouse.application.yml \ &> /var/log/feast-serving-warehouse.log & sleep 15 -tail -n10 /var/log/feast-serving-warehouse.log +tail -n100 /var/log/feast-serving-warehouse.log echo " ============================================================