Skip to content

Commit

Permalink
Redis Connector with Keda (#67)
Browse files Browse the repository at this point in the history
  • Loading branch information
@blackfly19
blackfly19 committed Sep 6, 2021
1 parent d4c68ea commit 617b5bb
Show file tree
Hide file tree
Showing 20 changed files with 875 additions and 64 deletions.
163 changes: 116 additions & 47 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name: Integration Test WF
on:
on:
push:
branches:
- master
Expand All @@ -10,10 +10,12 @@ env:
KIND_URL: https://kind.sigs.k8s.io/dl
KIND_VER: v0.8.1
KIND_ENV: kind-linux-amd64

jobs:
check:
runs-on: ubuntu-latest
outputs:
packages: ${{ steps.filter.outputs.changes }}
steps:
- name: Checkout the current repo
uses: actions/checkout@v1
Expand All @@ -27,45 +29,47 @@ jobs:
- 'rabbitmq-http-connector/**'
sqschanges:
- 'aws-sqs-http-connector/**'
kafka:
needs: check
if: contains(needs.check.outputs.packages, 'kafkachanges')
runs-on: ubuntu-latest
steps:
redischanges:
- 'redis-http-connector/**'
kafka:
needs: check
if: contains(needs.check.outputs.packages, 'kafkachanges')
runs-on: ubuntu-latest
steps:
- name: Install Kind Cluster
run: |
curl -Lo ./kind $KIND_URL/$KIND_VER/$KIND_ENV
chmod +x ./kind
sudo mv ./kind /usr/bin/
- name: Install KEDA using HELM
- name: Install KEDA using HELM
run: |
cd test/
./registry.sh
helm repo add kedacore https://kedacore.github.io/charts
helm repo update
kubectl create namespace keda
helm install keda kedacore/keda --namespace keda
- name: Create Docker Image for HTTP server
- name: Create Docker Image for HTTP server
run: |
cd test/server/
docker build -t localhost:5000/http-server:latest .
docker push localhost:5000/http-server:latest
kubectl apply -f server.yaml
kubectl wait pod -l app=http --for=condition=ready --timeout=-1s
kubectl wait pod -l app=http --for=condition=ready --timeout=-1s
- name: Create Docker Image for Kafka KEDA Connector
if: steps.filter.outputs.kafkachanges == 'true'
if: steps.filter.outputs.kafkachanges == 'true'
run: |
cd kafka-http-connector/
docker build -t localhost:5000/kafka-connector:latest .
docker push localhost:5000/kafka-connector:latest
- name: Create Kafka Docker Image for Consumer
if: steps.filter.outputs.kafkachanges == 'true'
if: steps.filter.outputs.kafkachanges == 'true'
run: |
cd kafka-http-connector/test/consumer/
docker build -t localhost:5000/consumer:latest .
docker push localhost:5000/consumer:latest
- name: Deploy Kafka cluster
if: steps.filter.outputs.kafkachanges == 'true'
if: steps.filter.outputs.kafkachanges == 'true'
run: |
cd kafka-http-connector/test/kubernetes/
kubectl create namespace kafka
Expand All @@ -78,14 +82,14 @@ jobs:
kubectl get pods -n kafka
kubectl wait pod -l app.kubernetes.io/name=zookeeper --for=condition=ready --timeout=-1s -n kafka
- name: Create Kafka topics
if: steps.filter.outputs.kafkachanges == 'true'
if: steps.filter.outputs.kafkachanges == 'true'
run: |
cd kafka-http-connector/test/kubernetes/
kubectl apply -f kafka-req-topic.yaml
kubectl apply -f kafka-err-topic.yaml
kubectl apply -f kafka-res-topic.yaml
- name: Deploy Kafka Keda Connector and Keda ScaledObject
if: steps.filter.outputs.kafkachanges == 'true'
if: steps.filter.outputs.kafkachanges == 'true'
run: |
cd kafka-http-connector/test/kubernetes/
kubectl apply -f keda-deployment.yml
Expand All @@ -94,33 +98,33 @@ jobs:
sleep 15s
kubectl get pods -n kafka
kubectl wait pod -l keda=kafka --for=condition=ready --timeout=30s -n kafka
kubectl apply -f Keda-ScaledObj.yml
kubectl apply -f Keda-ScaledObj.yml
- name: Produce Kafka messages Using Producer
if: steps.filter.outputs.kafkachanges == 'true'
if: steps.filter.outputs.kafkachanges == 'true'
run: |
cd kafka-http-connector/test/kubernetes/
kubectl apply -f kafka-produer.yaml -n kafka
kubectl wait -f kafka-produer.yaml --for=condition=complete --timeout=-1s -n kafka
kubectl delete job pi -n kafka
- name: Collect Kafka Consumer Messages
if: steps.filter.outputs.kafkachanges == 'true'
if: steps.filter.outputs.kafkachanges == 'true'
run: |
kubectl get pods -n kafka
cd kafka-http-connector/test/consumer/
kubectl apply -f consumer-deployment.yaml
kubectl wait pod -l app=consumer --for=condition=ready --timeout=-1s -n kafka
kubectl logs -l app=consumer --all-containers=true -n kafka
rabbitmq:
needs: check
if: contains(needs.check.outputs.packages, 'rabbitmqchanges')
runs-on: ubuntu-latest
steps:
kubectl logs -l app=consumer --all-containers=true -n kafka
rabbitmq:
needs: check
if: contains(needs.check.outputs.packages, 'rabbitmqchanges')
runs-on: ubuntu-latest
steps:
- name: Install Kind Cluster
run: |
curl -Lo ./kind $KIND_URL/$KIND_VER/$KIND_ENV
chmod +x ./kind
sudo mv ./kind /usr/bin/
- name: Install KEDA using HELM
- name: Install KEDA using HELM
run: |
cd test/
./registry.sh
Expand All @@ -129,95 +133,95 @@ jobs:
kubectl create namespace keda
helm install keda kedacore/keda --namespace keda
- name: Create Docker Image for Rabbitmq KEDA Connector
if: steps.filter.outputs.rabbitmqchanges == 'true'
if: steps.filter.outputs.rabbitmqchanges == 'true'
run: |
cd rabbitmq-http-connector/
docker build -t localhost:5000/rabbit-keda:latest .
docker push localhost:5000/rabbit-keda:latest
- name: Create Rabbitmq Docker Image for Publisher
if: steps.filter.outputs.rabbitmqchanges == 'true'
if: steps.filter.outputs.rabbitmqchanges == 'true'
run: |
cd rabbitmq-http-connector/test/publisher/
docker build -t localhost:5000/rabbit-publisher:latest .
docker push localhost:5000/rabbit-publisher:latest
- name: Create Rabbitmq Docker Image for Consumer
if: steps.filter.outputs.rabbitmqchanges == 'true'
if: steps.filter.outputs.rabbitmqchanges == 'true'
run: |
cd rabbitmq-http-connector/test/consumer/
docker build -t localhost:5000/rabbit-consumer:latest .
docker push localhost:5000/rabbit-consumer:latest
- name: Deploy Rabbitmq Deployment files
if: steps.filter.outputs.rabbitmqchanges == 'true'
if: steps.filter.outputs.rabbitmqchanges == 'true'
run: |
cd rabbitmq-http-connector/test/kubernetes/
kubectl create ns rabbits
kubectl apply -n rabbits -f rabbit-rbac.yaml
kubectl apply -n rabbits -f rabbit-configmap.yaml
kubectl apply -n rabbits -f rabbit-secret.yaml
kubectl apply -n rabbits -f rabbit-statefulset.yaml
bash rabbitmq-cluster-instances.sh
bash rabbitmq-cluster-instances.sh
- name: Publish Rabbitmq messages in a queue
if: steps.filter.outputs.rabbitmqchanges == 'true'
if: steps.filter.outputs.rabbitmqchanges == 'true'
run: |
cd rabbitmq-http-connector/test/publisher/
kubectl apply -f deployment.yaml
kubectl apply -f publisher-job.yaml
- name: Bring up the Rabbitmq consumer queue and Listen for incoming messages
if: steps.filter.outputs.rabbitmqchanges == 'true'
if: steps.filter.outputs.rabbitmqchanges == 'true'
run: |
cd rabbitmq-http-connector/test/consumer/
kubectl apply -f consumer-deployment.yaml
- name: Deploy Rabbitmq Keda Connector and Keda ScaledObject
if: steps.filter.outputs.rabbitmqchanges == 'true'
if: steps.filter.outputs.rabbitmqchanges == 'true'
run: |
cd rabbitmq-http-connector/test/kubernetes/
kubectl apply -f keda-deployment.yml
kubectl wait pod -l keda=rabbitmq --for=condition=ready --timeout=-1s -n rabbits
kubectl apply -f Keda-ScaledObj.yml
- name: Get Rabbitmq consumed messages from queue
if: steps.filter.outputs.rabbitmqchanges == 'true'
if: steps.filter.outputs.rabbitmqchanges == 'true'
run: |
sleep 10s
kubectl logs -n rabbits deployment.apps/rabbitmq-consumer
sqs:
needs: check
if: contains(needs.check.outputs.packages, 'sqschanges')
runs-on: ubuntu-latest
steps:
kubectl logs -n rabbits deployment.apps/rabbitmq-consumer
sqs:
needs: check
if: contains(needs.check.outputs.packages, 'sqschanges')
runs-on: ubuntu-latest
steps:
- name: Install Kind Cluster
run: |
curl -Lo ./kind $KIND_URL/$KIND_VER/$KIND_ENV
chmod +x ./kind
sudo mv ./kind /usr/bin/
- name: Install KEDA using HELM
- name: Install KEDA using HELM
run: |
cd test/
./registry.sh
helm repo add kedacore https://kedacore.github.io/charts
helm repo update
kubectl create namespace keda
- name: Create Docker Image for SQS KEDA Connector
if: steps.filter.outputs.sqschanges == 'true'
if: steps.filter.outputs.sqschanges == 'true'
run: |
cd aws-sqs-http-connector
docker build -t localhost:5000/aws-sqs-connector:latest .
docker push localhost:5000/aws-sqs-connector:latest
- name: Create SQS Docker image with AWS CLI in KEDA Connetor
if: steps.filter.outputs.sqschanges == 'true'
if: steps.filter.outputs.sqschanges == 'true'
run: |
cd aws-sqs-http-connector/test/kubernetes
docker build -t localhost:5000/aws-sqs-test-connector:latest .
docker push localhost:5000/aws-sqs-test-connector:latest
- name: Deploy SQS Keda Connector, Keda ScaledObject and Localstack
if: steps.filter.outputs.sqschanges == 'true'
if: steps.filter.outputs.sqschanges == 'true'
run: |
cd aws-sqs-http-connector/test/kubernetes
kubectl apply -f keda-deployment.yml
kubectl apply -f keda-scaledObj.yml
kubectl apply -f localstack.yaml
- name: Build and Deploy SQS test queue
if: steps.filter.outputs.sqschanges == 'true'
run: |
if: steps.filter.outputs.sqschanges == 'true'
run: |
cd aws-sqs-http-connector/test/validate
docker build -t localhost:5000/test-queue:latest .
docker push localhost:5000/test-queue:latest
Expand All @@ -226,7 +230,72 @@ jobs:
kubectl get pods
kubectl wait pod -l app=queue --for=condition=ready --timeout=200s
- name: Get SQS consumed messages from queue
if: steps.filter.outputs.sqschanges == 'true'
if: steps.filter.outputs.sqschanges == 'true'
run: |
sleep 30s
kubectl logs -l app=queue
redis:
needs: check
if: contains(needs.check.outputs.packages, 'redischanges')
runs-on: ubuntu-latest
steps:
- name: Checkout the current repo
uses: actions/checkout@v2
- name: Install Kind Cluster
run: |
curl -Lo ./kind $KIND_URL/$KIND_VER/$KIND_ENV
chmod +x ./kind
sudo mv ./kind /usr/bin/
- name: Install KEDA using HELM
run: |
cd test/
./registry.sh
helm repo add kedacore https://kedacore.github.io/charts
helm repo update
kubectl create namespace keda
helm install keda kedacore/keda --namespace keda
- name: Create Docker Image for Redis KEDA Connector
run: |
cd redis-http-connector/
docker build -t localhost:5000/redis-connector:latest .
docker push localhost:5000/redis-connector:latest
- name: Create Redis Docker Image for Publisher
run: |
cd redis-http-connector/test/publisher/
docker build -t localhost:5000/redis-publisher:latest .
docker push localhost:5000/redis-publisher:latest
- name: Create Redis Docker Image for Consumer
run: |
cd redis-http-connector/test/consumer/
docker build -t localhost:5000/redis-consumer:latest .
docker push localhost:5000/redis-consumer:latest
- name: Deploy Redis cluster
run: |
helm repo add ot-helm https://ot-container-kit.github.io/helm-charts/
kubectl create namespace redis-operator
helm upgrade redis-operator ot-helm/redis-operator --install --namespace redis-operator
sleep 30s
cd redis-http-connector/test/kubernetes
kubectl create namespace ot-operators
helm upgrade redis ot-helm/redis -f redis-values.yaml --install --namespace ot-operators
sleep 30s
kubectl get pods -n ot-operators
- name: Publish Redis messages in a queue
run: |
cd redis-http-connector/test/publisher/
kubectl apply -f publisher-deployment.yaml
- name: Bring up the Redis consumer queue and Listen for incoming messages
run: |
cd redis-http-connector/test/consumer/
kubectl apply -f consumer-deployment.yaml
- name: Deploy Redis Keda Connector and Keda ScaledObject
run: |
cd redis-http-connector/test/kubernetes/
kubectl apply -f keda-deployment.yaml
sleep 30s
kubectl apply -f Keda-ScaledObj.yaml
sleep 30s
- name: Get Redis consumed messages from queue
run: |
sleep 30s
kubectl logs deployment.apps/consumer-deployment
36 changes: 19 additions & 17 deletions .github/workflows/release.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ on:
branches:
- master
paths:
- '**/version'
- "**/version"
jobs:
check:
runs-on: ubuntu-latest
Expand All @@ -29,25 +29,27 @@ jobs:
- 'aws-kinesis-http-connector/version'
gcp-pubsub-http-connector:
- 'gcp-pubsub-http-connector/version'
redis-http-connector:
- 'redis-http-connector/version'
run-docker-build-push-action:
needs: check
strategy:
matrix:
run: ${{ fromJSON(needs.check.outputs.packages) }}
run: ${{ fromJSON(needs.check.outputs.packages) }}
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@v2
- name: get version
run: cd ${{ matrix.run }}; version=$(cat version);echo "version=$version" >> $GITHUB_ENV
- name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and Push the docker image
uses: docker/build-push-action@v2
with:
context: ${{ matrix.run }}
tags: fission/keda-${{ matrix.run }}:latest,fission/keda-${{ matrix.run }}:${{env.version}}
push: true
- name: Checkout sources
uses: actions/checkout@v2
- name: get version
run: cd ${{ matrix.run }}; version=$(cat version);echo "version=$version" >> $GITHUB_ENV
- name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and Push the docker image
uses: docker/build-push-action@v2
with:
context: ${{ matrix.run }}
tags: fission/keda-${{ matrix.run }}:latest,fission/keda-${{ matrix.run }}:${{env.version}}
push: true

0 comments on commit 617b5bb

Please sign in to comment.