GitHub Action for integration tests (base infrastructure) (feast-dev#4)

Builds, tests and runs Docker container for Feast core as well as needed services (Kafka, Redis and PostgreSQL).

Uses a self-hosted agent VM (deployed on Azure) to speed up build thanks to Docker layer caching.

The Docker Action automatically runs whenever a PR is made.

.github/workflows/integration-tests.yml

@@ -0,0 +1,90 @@
+name: ci
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the master branch
+  push:
+    branches:
+    - master
+  pull_request:
+    branches:
+    - master
+
+jobs:
+
+  build-and-test:
+
+    # Run the job on a self-hosted agent VM. The VM will cache Docker layers, which considerably speeds up the build.
+    runs-on: self-hosted
+    steps:
+
+    # Delete any files left from the previous build, including files owned by root created by Docker in mounted volumes.
+    - name: Clean current directory
+      run: shopt -s dotglob; sudo rm -rf *
+
+    # Checkout sources from Git
+    - uses: actions/checkout@v2
+
+    # Build base CI image, with dependencies such as Go and Python.
+    # This image seldom changes, so thanks to layer caching this usually completes in seconds.
+    - name: Build CI image
+      uses: docker/build-push-action@v1
+      with:
+        username: ${{ secrets.CONTAINERREGISTRY_USERNAME }}
+        password: ${{ secrets.CONTAINERREGISTRY_PASSWORD }}
+        repository: ${{ secrets.CONTAINERREGISTRY_IMAGENAMEBASE }}-ci
+        registry: ${{ secrets.CONTAINERREGISTRY_URL }}
+        dockerfile: infra/docker/ci/Dockerfile
+        tags: v${{ github.sha }}
+
+
+    # Cache the Maven repository across runs.
+    - name: Cache Maven repository
+      id: cache-maven
+      uses: actions/cache@v1
+      with:
+        path: .m2
+        key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          ${{ runner.os }}-maven-
+
+    # Generate the Maven cache if needed by running a throwaway build of Feast.
+    - name: Generate Maven cache
+      if: steps.cache-maven.outputs.cache-hit != 'true'
+      run: docker run --rm --user $(id -u):$(id -g) -v $PWD:/build $IMAGE bash -c "cd /build && mvn -Dmaven.repo.local=/build/.m2/repository -Dgpg.skip=true -B verify -DskipTests"
+      env:
+        IMAGE: ${{ secrets.CONTAINERREGISTRY_URL }}/${{ secrets.CONTAINERREGISTRY_IMAGENAMEBASE }}-ci:v${{ github.sha }}
+
+    # "docker-compose up --build" builds and starts docker images.
+    # Images fetched from a repository, such as Kafka, will not be built, so this
+    # step will only build the Feast Core image.
+    # After this step, the specified containers have been started and Docker Core
+    # has successfully connected to its required services (such as PostgreSQL).
+    - name: Start local Feast environment
+      run: |
+        docker-compose -f infra/docker-compose/docker-compose.yml up -d --build core redis kafka db
+        for i in {1..12}; do nc -zv localhost 6565 && break; echo "Waiting for Feast Core to come online on port 6565..."; sleep 10; done
+      env:
+        COMPOSE_PROJECT_NAME: feast
+        FEAST_VERSION: v${{ github.sha }}
+        FEAST_CORE_IMAGE: ${{ secrets.CONTAINERREGISTRY_URL }}/${{ secrets.CONTAINERREGISTRY_IMAGENAMEBASE }}-core
+        FEAST_CORE_CONFIG: direct-runner.yml
+        FEAST_SERVING_IMAGE: gcr.io/kf-feast/feast-serving
+        FEAST_ONLINE_SERVING_CONFIG: online-serving.yml
+        FEAST_ONLINE_STORE_CONFIG: redis-store.yml
+
+    # Login to Azure Container Registry.
+    - name: Login to Azure Container Registry
+      uses: azure/docker-login@v1
+      with:
+        login-server: ${{ secrets.CONTAINERREGISTRY_URL }}
+        username: ${{ secrets.CONTAINERREGISTRY_USERNAME }}
+        password: ${{ secrets.CONTAINERREGISTRY_PASSWORD }}
+
+    # Promote successfully tested container to Azure Container Registry.
+    - name: Push image to ACR
+      run: |
+        docker-compose -f infra/docker-compose/docker-compose.yml push core
+      env:
+        FEAST_VERSION: v${{ github.sha }}
+        FEAST_CORE_IMAGE: ${{ secrets.CONTAINERREGISTRY_URL }}/${{ secrets.CONTAINERREGISTRY_IMAGENAMEBASE }}-core

docs/cicd/README.md

@@ -0,0 +1,54 @@
+# FarFetch CI setup
+
+The CI setup ensures that PRs on GitHub are automatically tested before being merging to *master*. The CI jobs defined under 
+
+[.github/workflows]: ../../.github/workflows
+
+run compilation, unit and end-to-end tests using GitHub Workflows.
+
+ ## Agent configuration
+
+1. Spin an Azure VM with Ubuntu 18.04. Choose e.g. a B4ms instance (burstable VM with 4 cores)
+2. Install prerequisites:
+
+
+```
+sudo apt update
+sudo apt install docker.io
+sudo usermod -aG docker $USER
+
+sudo curl -L "https://github.com/docker/compose/releases/download/1.25.4/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+sudo chmod +x docker-compose
+```
+
+3. Follow the actions to [add a self-hosted runner to your repository](https://help.github.com/en/actions/hosting-your-own-runners/adding-self-hosted-runners)
+4. Follow the actions to [configure the self-hosted runner as a service](https://help.github.com/en/actions/hosting-your-own-runners/configuring-the-self-hosted-runner-application-as-a-service)
+
+
+
+ ## One-time CI configuration
+
+Create an Azure Container Registry named *acrffgithubci*. Make sure to Enable Admin user.
+
+![Deploy ACR](deploy-acr.png)
+
+Once the ACR instance is deployed, fetch the password from the Access keys section:
+
+![ACR password](acr-password.png)
+
+Populate the GitHub secrets settings with the ACR configuration:
+
+![](create-secrets.png)
+
+- CONTAINERREGISTRY_IMAGENAMEBASE=farfetchfeast
+- CONTAINERREGISTRY_PASSWORD=<ACR password>
+- CONTAINERREGISTRY_URL=acrffgithubci.azurecr.io
+- CONTAINERREGISTRY_USERNAME=acrffgithubci
+
+ ## CI jobs
+
+The integration test job deploys the application components using Docker-compose. Later on, it will be extended to run end-to-end test scenarios.
+
+![Integration test job](integration-test-job.png)
+
+

infra/docker-compose/docker-compose.yml

@@ -2,6 +2,9 @@ version: "3.7"
 
 services:
   core:
+    build:
+      context: ../..
+      dockerfile: infra/docker/core/Dockerfile
     image: ${FEAST_CORE_IMAGE}:${FEAST_VERSION}
     volumes:
       - ./core/${FEAST_CORE_CONFIG}:/etc/feast/application.yml
@@ -22,6 +25,9 @@ services:
       - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yaml
 
   online-serving:
+    build:
+      context: ../..
+      dockerfile: infra/docker/serving/Dockerfile
     image: ${FEAST_SERVING_IMAGE}:${FEAST_VERSION}
     volumes:
       - ./serving/${FEAST_ONLINE_SERVING_CONFIG}:/etc/feast/application.yml
@@ -39,6 +45,9 @@ services:
       - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml
 
   batch-serving:
+    build:
+      context: ../..
+      dockerfile: infra/docker/serving/Dockerfile
     image: ${FEAST_SERVING_IMAGE}:${FEAST_VERSION}
     volumes:
       - ./serving/${FEAST_BATCH_SERVING_CONFIG}:/etc/feast/application.yml

infra/docker/ci/Dockerfile

@@ -41,4 +41,8 @@ RUN PROTOC_ZIP=protoc-${PROTOC_VERSION}-linux-x86_64.zip && \
     go get gopkg.in/russross/blackfriday.v2 && \
 	  git clone https://github.com/istio/tools/ && \
 	  cd tools/cmd/protoc-gen-docs && \
-	  go build && mkdir -p $HOME/bin && cp protoc-gen-docs $HOME/bin
+	  go build && mkdir -p $HOME/bin && cp protoc-gen-docs $HOME/bin
+
+COPY Makefile /build/Makefile
+COPY sdk/python/requirements-ci.txt /build/sdk/python/requirements-ci.txt
+RUN cd /build && make install-python-ci-dependencies install-go-ci-dependencies && rm -rf /build