Added Tolerations and Node Affinity #23
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build | |
| # Controls when the action will run. | |
| on: | |
| # pull_request: | |
| push: | |
| jobs: | |
| deploy: | |
| name: K8s QuickStart CI test | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Set env and tools | |
| run: | | |
| echo "TESTCLUSTERNAME=k8s-gha-test-$(date +%s)" >> $GITHUB_ENV | |
| echo "TESTRUNBRANCH=${GITHUB_REF##*/}" >> $GITHUB_ENV | |
| # | |
| sudo gem install yaml-lint | |
| sudo snap install kubectl --classic | |
| kubectl version --client | |
| curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash # setup Helm 3 | |
| docker ps | |
| go version | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v0.6.0 | |
| with: | |
| project_id: ${{ secrets.GCP_PROJECT_ID }} | |
| service_account_key: ${{ secrets.GCP_SA_KEY }} | |
| export_default_credentials: true | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v1 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Linting yaml files and chart | |
| run: | | |
| yaml-lint -n pubsubplus/*.yaml | |
| helm lint pubsubplus | |
| - name: Run Whitesource Action | |
| uses: SolaceDev/Mend-Scan-GHA@v1.0.0 | |
| with: | |
| wssURL: https://saas.whitesourcesoftware.com/agent | |
| apiKey: ${{ secrets.WSS_API_KEY }} | |
| productName: 'pubsubplus-kubernetes-helm' | |
| projectName: 'pubsubplus-kubernetes-helm' | |
| configFile: 'ci/whitesource/whitesource-agent.config' | |
| - name: Setup K8s env in GKE | |
| run: | | |
| gcloud components install gke-gcloud-auth-plugin --quiet | |
| gcloud components update | |
| export USE_GKE_GCLOUD_AUTH_PLUGIN=True | |
| mkdir gke_test; pushd gke_test | |
| wget https://raw.githubusercontent.com/SolaceProducts/solace-gke-quickstart/master/scripts/create_cluster.sh | |
| chmod +x create_cluster.sh | |
| ./create_cluster.sh -z us-east4-a,us-east4-b,us-east4-c -c $TESTCLUSTERNAME -i ubuntu_containerd -m e2-standard-4 | |
| gcloud container clusters get-credentials $TESTCLUSTERNAME --zone us-east4-a --project capable-stream-180018 | |
| popd | |
| kubectl get statefulset,svc,pods,pvc,pv | |
| - name: Setup pod modifier | |
| run: | | |
| pushd solace-pod-modifier-admission-plugin | |
| make image-build image-push IMAGE=${{ secrets.DOCKERHUB_TEST_IMAGE }} | |
| make deploy IMAGE=${{ secrets.DOCKERHUB_TEST_IMAGE }} | |
| sleep 2 | |
| timeout 20 bash -c 'while ! kubectl get pods -n solace-pod-modifier | grep Running ; do sleep 1; done' | |
| timeout 20 bash -c 'while ! kubectl get MutatingWebhookConfiguration | grep pod-modifier.solace.com ; do sleep 1; done' | |
| kubectl label namespace default pod-modifier.solace.com=enabled # prep namespace for use | |
| popd | |
| - name: Deploy HA broker and test | |
| run: | | |
| REPO=solace/solace-pubsub-standard | |
| TAG=latest | |
| openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout tls.key -out tls.crt -subj "/CN=*" | |
| kubectl create secret tls test-tls --key="tls.key" --cert="tls.crt" | |
| helm install my-release pubsubplus --set solace.size=dev,solace.redundancy=true,solace.podDisruptionBudgetForHA=true,solace.podModifierEnabled=true,tls.enabled=true,tls.serverCertificatesSecret=test-tls,solace.usernameAdminPassword=admin,image.repository=$REPO,image.tag=$TAG | |
| kubectl get statefulset,svc,pods,pvc,pv --show-labels | |
| echo "Waiting for broker to become active" | |
| sleep 60; kubectl describe nodes | |
| until kubectl get pods --show-labels | grep pubsubplus-0 | grep -m 1 -E '1/1'; do sleep 10; done | |
| until kubectl get pods --show-labels | grep pubsubplus-1 | grep -m 1 -E '1/1'; do sleep 10; done | |
| until kubectl get pods --show-labels | grep pubsubplus-2 | grep -m 1 -E '1/1'; do sleep 10; done | |
| until kubectl get pods --show-labels | grep pubsubplus- | grep -m 1 -E 'active=true'; do sleep 10; done | |
| kubectl get pods -o yaml | grep "memory: 1965Mi" # test small monitor memory | |
| kubectl get pvc | grep 2Gi # test small monitor storage | |
| helm test my-release | grep Phase | grep Succeeded | |
| kubectl get statefulset,svc,pods,pvc,pv --show-labels | |
| bash -c 'if [[ `kubectl get po --show-labels | grep -c "1/1"` -ne 3 ]]; then echo "Some pods are not ready!"; kubectl get po --show-labels; exit 1; fi' | |
| export url="$(kubectl get statefulset,svc,pods,pvc,pv --show-labels | grep LoadBalancer | awk '{print $4}')"; echo $url | |
| curl -O https://sftp.solace.com/download/SDKPERF_C_LINUX64 | |
| tar -xvf SDKPERF_C_LINUX64 | |
| pubSubTools/sdkperf_c -cip=tcp://$url:55555 -mn=10000 -mr=0 -ptl=t1 -stl=t1 | grep "Total Messages" | |
| pubSubTools/sdkperf_c -cip=tcps://$url:55443 -mn=10000 -mr=0 -ptl=t1 -stl=t1 | grep "Total Messages" | |
| sleep 30 | |
| curl -k -sS -u admin:admin https://$url:1943/SEMP -d "<rpc><show><redundancy></redundancy></show></rpc>" | |
| curl -k -sS -u admin:admin https://$url:1943/SEMP -d "<rpc><show><config-sync></config-sync></show></rpc>" | |
| if [[ -z `curl -sS -u admin:admin http://$url:8080/SEMP -d "<rpc><show><config-sync></config-sync></show></rpc>" | grep "<oper-status>Up</oper-status>"` ]] ; then echo "config-sync not up!"; exit 1; fi | |
| helm list | |
| - name: Upgrade HA broker and test | |
| run: | | |
| REPO=solace/solace-pubsub-standard | |
| # grab a tag from Docker Hub that has the same SHA as "latest", so upgrade is easy | |
| DOCKERHUBRESULTS=`curl --silent "https://hub.docker.com/v2/repositories/$REPO/tags?page_size=1000" | jq -r '.results[] | "\(.digest) \(.name)"' | sort` | |
| SHA=`echo "$DOCKERHUBRESULTS" | grep latest | awk '{print $1;}'` | |
| UPGRADETAG=`echo "$DOCKERHUBRESULTS" | grep $SHA | head -n 1 | awk '{print $2;}'` | |
| helm upgrade my-release pubsubplus --set solace.size=dev,solace.redundancy=true,solace.podDisruptionBudgetForHA=true,solace.podModifierEnabled=true,tls.enabled=true,tls.serverCertificatesSecret=test-tls,solace.usernameAdminPassword=admin,image.repository=$REPO,image.tag=$UPGRADETAG,storage.useStorageGroup=true | |
| kubectl get statefulset,svc,pods,pvc,pv --show-labels | |
| echo "Waiting for broker to become active after upgrade" | |
| sleep 20; kubectl describe nodes | |
| statefulset_name=$(kubectl get statefulset | grep pubsubplus | awk '{print $1}') | |
| until kubectl rollout status statefulset $statefulset_name -w | grep "rolling update complete"; do sleep 10; done | |
| until kubectl get pods --show-labels | grep pubsubplus-0 | grep -m 1 -E '1/1'; do sleep 10; done | |
| until kubectl get pods --show-labels | grep pubsubplus-1 | grep -m 1 -E '1/1'; do sleep 10; done | |
| until kubectl get pods --show-labels | grep pubsubplus-2 | grep -m 1 -E '1/1'; do sleep 10; done | |
| until kubectl get pods --show-labels | grep pubsubplus- | grep -m 1 -E 'active=true'; do sleep 10; done | |
| helm test my-release | grep Phase | grep Succeeded | |
| kubectl get statefulset,svc,pods,pvc,pv --show-labels | |
| bash -c 'if [[ `kubectl get po --show-labels | grep -c "1/1"` -ne 3 ]]; then echo "Some pods are not ready!"; kubectl get po --show-labels; exit 1; fi' | |
| export url="$(kubectl get statefulset,svc,pods,pvc,pv --show-labels | grep LoadBalancer | awk '{print $4}')"; echo $url | |
| pubSubTools/sdkperf_c -cip=tcp://$url:55555 -mn=10000 -mr=0 -ptl=t1 -stl=t1 | grep "Total Messages" | |
| pubSubTools/sdkperf_c -cip=tcps://$url:55443 -mn=10000 -mr=0 -ptl=t1 -stl=t1 | grep "Total Messages" | |
| sleep 10 | |
| curl -k -sS -u admin:admin https://$url:1943/SEMP -d "<rpc><show><redundancy></redundancy></show></rpc>" | |
| curl -k -sS -u admin:admin https://$url:1943/SEMP -d "<rpc><show><config-sync></config-sync></show></rpc>" | |
| if [[ -z `curl -sS -u admin:admin http://$url:8080/SEMP -d "<rpc><show><config-sync></config-sync></show></rpc>" | grep "<oper-status>Up</oper-status>"` ]] ; then echo "config-sync not up!"; exit 1; fi | |
| helm list | |
| helm delete $(helm list | grep deployed | awk '{print $1}') | |
| kubectl delete secret test-tls | |
| kubectl delete pvc --all | |
| - name: Test HA broker Toleration | |
| run: | | |
| REPO=solace/solace-pubsub-standard | |
| TAG=latest | |
| kubectl get nodes | |
| #create taint for first node | |
| nodesAll=$(kubectl get nodes --output name) && firstNode=`echo "${nodesAll}" | head -1` | |
| kubectl taint nodes $firstNode scheduleBroker=no:NoSchedule | |
| #confirm node taint | |
| echo "$firstNode has been tainted" | |
| kubectl describe $firstNode | grep scheduleBroker | |
| openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout tls.key -out tls.crt -subj "/CN=*" | |
| kubectl create secret tls test-tls --key="tls.key" --cert="tls.crt" | |
| helm install my-release-toleration pubsubplus --set "solace.tolerations[0].key=scheduleBroker,solace.tolerations[0].operator=Equal,solace.tolerations[0].value=no,solace.tolerations[0].effect=NoSchedule,solace.size=dev,solace.redundancy=true,solace.podDisruptionBudgetForHA=true,solace.podModifierEnabled=true,tls.enabled=true,tls.serverCertificatesSecret=test-tls,solace.usernameAdminPassword=admin,image.repository=$REPO,image.tag=$TAG" | |
| kubectl get statefulset,svc,pods,pvc,pv --show-labels | |
| echo "Waiting for broker to become active" | |
| sleep 60; kubectl describe nodes | |
| until kubectl get pods --show-labels | grep pubsubplus-0 | grep -m 1 -E '1/1'; do sleep 10; done | |
| until kubectl get pods --show-labels | grep pubsubplus-1 | grep -m 1 -E '1/1'; do sleep 10; done | |
| until kubectl get pods --show-labels | grep pubsubplus-2 | grep -m 1 -E '1/1'; do sleep 10; done | |
| until kubectl get pods --show-labels | grep pubsubplus- | grep -m 1 -E 'active=true'; do sleep 10; done | |
| bash -c 'if [[ `kubectl get po --show-labels | grep -c "1/1"` -ne 3 ]]; then echo "Some pods are not ready!"; kubectl get po --show-labels; exit 1; fi' | |
| #confirm broker deployment meets toleration requirements | |
| echo "get node confirmation of no resources" | |
| kubectl get pods --all-namespaces -o wide --field-selector spec.nodeName=$firstNode | |
| sleep 30 | |
| helm list | |
| helm delete $(helm list | grep deployed | awk '{print $1}') | |
| kubectl taint nodes $firstNode scheduleBroker=no:NoSchedule- | |
| kubectl delete secret test-tls | |
| kubectl delete pvc --all | |
| - name: Create chart variants | |
| run: | | |
| bash docs/helm-charts/create-chart-variants.sh; # Create chart variants | |
| helm lint pubsubplus | |
| helm install --generate-name pubsubplus --dry-run | |
| helm lint pubsubplus-ha | |
| helm install --generate-name pubsubplus-ha --dry-run | |
| helm lint pubsubplus-dev | |
| helm install --generate-name pubsubplus-dev --dry-run | |
| helm lint pubsubplus-openshift | |
| helm install --generate-name pubsubplus-openshift --dry-run | |
| helm lint pubsubplus-openshift-ha | |
| helm install --generate-name pubsubplus-openshift-ha --dry-run | |
| helm lint pubsubplus-openshift-dev | |
| helm install --generate-name pubsubplus-openshift-dev --dry-run | |
| - name: Publish artifacts | |
| run: | | |
| # Two groups of Helm repos are created: | |
| # 1 - for general Helm charts that are hosted by Solace from gh-pages | |
| # 2 - for OpenShift variants that will be further submitted to OpenShift repo | |
| git config --global user.name "GitHub Actions Automation" | |
| git config --global user.email "<>" | |
| mkdir gh-pages; # Now update gh-pages | |
| if [ ${{ github.ref }} == 'refs/heads/master' ] && [ ${{ github.repository_owner }} == 'SolaceProducts' ] ; then | |
| echo "Using master on SolaceProducts" | |
| git clone --quiet --branch=gh-pages https://${{ secrets.GH_TOKEN }}@github.com/SolaceProducts/pubsubplus-kubernetes-helm-quickstart gh-pages > /dev/null 2>&1 | |
| rm -rf gh-pages/helm-charts-openshift; mkdir -p gh-pages/helm-charts-openshift | |
| mv pubsubplus-openshift-*.tgz gh-pages/helm-charts-openshift/ | |
| helm repo index gh-pages/helm-charts-openshift/ --url https://solaceproducts.github.io/pubsubplus-kubernetes-helm-quickstart/helm-charts-openshift | |
| mv pubsubplus-*.tgz gh-pages/helm-charts/ | |
| helm repo index gh-pages/helm-charts/ --url https://solaceproducts.github.io/pubsubplus-kubernetes-helm-quickstart/helm-charts | |
| pushd gh-pages | |
| git add -f . | |
| git commit -m "Latest helm chart updates on successful gha-test build ${{ github.run_number }} auto-pushed to gh-pages" | |
| git remote add origin-pages https://${{ secrets.GH_TOKEN }}@github.com/SolaceProducts/pubsubplus-kubernetes-helm-quickstart.git > /dev/null 2>&1 | |
| git push --quiet --set-upstream origin-pages gh-pages | |
| popd | |
| echo "Updated and pushed GH pages!" | |
| elif [ ${{ github.ref }} != 'refs/heads/gh-pages' ] && [ ${{ github.repository_owner }} != 'SolaceProducts' ] && [[ ${{ github.ref }} =~ .*"refs/heads/v".* ]] ; then | |
| echo "Using $TESTRUNBRANCH on ${{ github.repository_owner }}" | |
| git clone --quiet --branch=gh-pages https://${{ secrets.GH_TOKEN }}@github.com/${{ github.repository }} gh-pages > /dev/null 2>&1 | |
| rm -rf gh-pages/helm-charts-openshift; mkdir -p gh-pages/helm-charts-openshift | |
| mv pubsubplus-openshift-*.tgz gh-pages/helm-charts-openshift/ | |
| helm repo index gh-pages/helm-charts-openshift/ --url https://solacedev.github.io/pubsubplus-kubernetes-helm-quickstart/helm-charts-openshift | |
| mv pubsubplus-*.tgz gh-pages/helm-charts/ | |
| helm repo index gh-pages/helm-charts/ --url https://solacedev.github.io/pubsubplus-kubernetes-helm-quickstart/helm-charts | |
| pushd gh-pages | |
| git add -f . | |
| git commit -m "Latest helm chart updates on successful gha-test build ${{ github.run_number }} auto-pushed to gh-pages" | |
| git remote add origin-pages https://${{ secrets.GH_TOKEN }}@github.com/${{ github.repository }}.git > /dev/null 2>&1 | |
| git push --quiet --set-upstream origin-pages gh-pages | |
| popd | |
| echo "Updated and pushed GH pages!" | |
| fi | |
| - name: Delete test resources (Cleanup) | |
| if: ${{ always() }} | |
| run: | | |
| gcloud container clusters delete $TESTCLUSTERNAME --quiet --zone us-east4-a | |
| gcloud compute disks list | grep gha-test | sed 1d $rpt | while read -r a b c; do gcloud compute disks delete $a --zone $b --quiet; done |