diff --git a/.github/workflows/build-tools.yml b/.github/workflows/build-tools.yml index 1f52232b72d..6f43e822491 100644 --- a/.github/workflows/build-tools.yml +++ b/.github/workflows/build-tools.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v2 with: fetch-depth: 1 diff --git a/.github/workflows/fossa.yml b/.github/workflows/fossa.yml index 4815a70953d..aac01cc404a 100644 --- a/.github/workflows/fossa.yml +++ b/.github/workflows/fossa.yml @@ -11,7 +11,7 @@ jobs: # build-tools is built from ../../tools/build-tools.Dockerfile container: ghcr.io/kedacore/build-tools:main steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 - uses: actions/setup-go@v3 with: go-version: "^1.17" diff --git a/.github/workflows/main-build.yml b/.github/workflows/main-build.yml index 26c9cee4980..84a1c3e4965 100644 --- a/.github/workflows/main-build.yml +++ b/.github/workflows/main-build.yml @@ -3,9 +3,10 @@ on: push: branches: - main +concurrency: e2e-tests jobs: - build: - name: build + validate: + name: Validate runs-on: ubuntu-latest permissions: contents: read @@ -16,7 +17,7 @@ jobs: container: ghcr.io/kedacore/build-tools:main steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v2 with: fetch-depth: 1 @@ -53,11 +54,8 @@ jobs: # Server address of Docker registry. If not set then will default to Docker Hub registry: ghcr.io - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Publish on GitHub Container Registry - run: make publish-multiarch + run: make publish # https://github.com/sigstore/cosign-installer - name: Install Cosign @@ -72,19 +70,6 @@ jobs: # against the sigstore community Fulcio instance. run: make sign-images - validate: - needs: build - name: validate - runs-on: ubuntu-latest - # build-tools is built from ../../tools/build-tools.Dockerfile - container: ghcr.io/kedacore/build-tools:main - concurrency: e2e-tests - steps: - - name: Check out code - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - name: Run end to end tests env: AZURE_APP_INSIGHTS_APP_ID: ${{ secrets.AZURE_APP_INSIGHTS_APP_ID }} @@ -120,9 +105,3 @@ jobs: - name: Delete all e2e related namespaces if: ${{ always() }} run: make e2e-test-clean - env: - AZURE_RESOURCE_GROUP: ${{ secrets.AZURE_RESOURCE_GROUP }} - AZURE_SP_ID: ${{ secrets.AZURE_SP_ID }} - AZURE_SP_KEY: ${{ secrets.AZURE_SP_KEY }} - AZURE_SP_TENANT: ${{ secrets.AZURE_SP_TENANT }} - AZURE_SUBSCRIPTION: ${{ secrets.AZURE_SUBSCRIPTION }} diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml index 83a969f8b6c..852ce6accb7 100644 --- a/.github/workflows/nightly-e2e.yml +++ b/.github/workflows/nightly-e2e.yml @@ -11,7 +11,7 @@ jobs: container: ghcr.io/kedacore/build-tools:main steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v2 with: fetch-depth: 1 @@ -50,9 +50,3 @@ jobs: - name: Delete all e2e related namespaces if: ${{ always() }} run: make e2e-test-clean - env: - AZURE_RESOURCE_GROUP: ${{ secrets.AZURE_RESOURCE_GROUP }} - AZURE_SP_ID: ${{ secrets.AZURE_SP_ID }} - AZURE_SP_KEY: ${{ secrets.AZURE_SP_KEY }} - AZURE_SP_TENANT: ${{ secrets.AZURE_SP_TENANT }} - AZURE_SUBSCRIPTION: ${{ secrets.AZURE_SUBSCRIPTION }} diff --git a/.github/workflows/pr-e2e.yml b/.github/workflows/pr-e2e.yml index c15b9a562aa..ea78dac3b04 100644 --- a/.github/workflows/pr-e2e.yml +++ b/.github/workflows/pr-e2e.yml @@ -35,7 +35,7 @@ jobs: concurrency: pr-e2e-tests if: needs.check.outputs.run-e2e == '1' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 - name: Checkout Pull Request env: @@ -110,13 +110,6 @@ jobs: - name: Delete all e2e related namespaces if: ${{ always() }} run: make e2e-test-clean - env: - AZURE_RESOURCE_GROUP: ${{ secrets.AZURE_RESOURCE_GROUP }} - AZURE_SP_ID: ${{ secrets.AZURE_SP_ID }} - AZURE_SP_KEY: ${{ secrets.AZURE_SP_KEY }} - AZURE_SP_TENANT: ${{ secrets.AZURE_SP_TENANT }} - AZURE_SUBSCRIPTION: ${{ secrets.AZURE_SUBSCRIPTION }} - TEST_CLUSTER_NAME: keda-pr-run - name: React to comment with success uses: dkershner6/reaction-action@v1 diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 900d81d6aea..77102b8cf4f 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -16,7 +16,7 @@ jobs: run: apt update && apt install -y make ca-certificates gcc - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v2 with: fetch-depth: 1 @@ -66,7 +66,7 @@ jobs: run: apt update && apt install -y make ca-certificates gcc - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v2 with: fetch-depth: 1 @@ -100,49 +100,11 @@ jobs: - name: Test run: make test - validate-dockerfiles: - name: Validate Dockerfiles - runs-on: ubuntu-latest - container: ghcr.io/kedacore/build-tools:main - steps: - - name: Check out code - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - dockerfiles: - - 'Dockerfile*' - - - name: Build images - if: steps.filter.outputs.dockerfiles == 'true' - run: make docker-build - - validate-build-tools: - name: Validate build-tools - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - tools: - - 'tools/**' - - - name: Build tools - if: steps.filter.outputs.tools == 'true' - run: make docker-build-tools - statics: name: Static Checks runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 - uses: actions/setup-python@v3.0.0 - uses: actions/setup-go@v3 with: @@ -163,7 +125,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v2 with: # We must fetch at least the immediate parents so that if this is # a pull request then we can checkout the head. diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index eaf3567feb0..9d0f036013d 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -16,7 +16,7 @@ jobs: container: ghcr.io/kedacore/build-tools:main steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v2 with: fetch-depth: 1 @@ -59,11 +59,8 @@ jobs: env: VERSION: ${{ steps.get_version.outputs.VERSION }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Publish KEDA images on GitHub Container Registry - run: make publish-multiarch + run: make publish env: VERSION: ${{ steps.get_version.outputs.VERSION }} diff --git a/.github/workflows/v1-build.yml b/.github/workflows/v1-build.yml index 52c2d2c7eb4..d4c482b2c9d 100644 --- a/.github/workflows/v1-build.yml +++ b/.github/workflows/v1-build.yml @@ -11,7 +11,7 @@ jobs: container: kedacore/build-tools:v1 steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v2 with: fetch-depth: 1 diff --git a/CHANGELOG.md b/CHANGELOG.md index a2d0d628099..fdc6d957db2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,9 +34,11 @@ ### Improvements +- **Azure Pipelines:** Added the ability to support ADO demands by referencing a master template ([#2707](https://github.com/kedacore/keda/pull/2707)) - **Azure Queue:** Don't call Azure queue GetProperties API unnecessarily ([#2613](https://github.com/kedacore/keda/pull/2613)) - **Datadog Scaler:** Validate query to contain `{` to prevent panic on invalid query ([#2625](https://github.com/kedacore/keda/issues/2625)) - **Kafka Scaler** Make "disable" a valid value for tls auth parameter ([#2608](https://github.com/kedacore/keda/issues/2608)) +- **Azure Pipelines:** Added the ability to support ADO demands by referencing a master template ([#2707](https://github.com/kedacore/keda/pull/2707)) ### Breaking Changes @@ -538,3 +540,5 @@ None. ### Other None. + + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 55c24f68448..5eb028c46ca 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,7 +54,6 @@ Here are some guidelines to follow: - Always use `General: ` or `: ` as a prefix and sort them alphabetically - General changes, however, should always be at the top - Entries should always follow the `: (#)` where `` is preferrably the ID of an issue, otherwise a PR is OK. -- New scalers should use `General:` and use this template: `**General:** Introduce new XXXXXX Scaler ([#ISSUE](https://github.com/kedacore/keda/issues/ISSUE))` ## Including Documentation Changes diff --git a/Dockerfile b/Dockerfile index 08211c032ce..0aed704d260 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,5 @@ # Build the manager binary -ARG BUILDPLATFORM=linux/amd64 -FROM --platform=$BUILDPLATFORM golang:1.17.3 AS builder +FROM golang:1.17.3 as builder ARG BUILD_VERSION=main ARG GIT_COMMIT=HEAD @@ -27,10 +26,7 @@ COPY controllers/ controllers/ COPY pkg/ pkg/ # Build -# https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/ -ARG TARGETOS=linux -ARG TARGETARCH=amd64 -RUN VERSION=${BUILD_VERSION} GIT_COMMIT=${GIT_COMMIT} GIT_VERSION=${GIT_VERSION} TARGET_OS=$TARGETOS ARCH=$TARGETARCH make manager +RUN VERSION=${BUILD_VERSION} GIT_COMMIT=${GIT_COMMIT} GIT_VERSION=${GIT_VERSION} make manager # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details diff --git a/Dockerfile.adapter b/Dockerfile.adapter index 68304867a90..35503027b07 100644 --- a/Dockerfile.adapter +++ b/Dockerfile.adapter @@ -1,6 +1,5 @@ # Build the adapter binary -ARG BUILDPLATFORM=linux/amd64 -FROM --platform=$BUILDPLATFORM golang:1.17.3 as builder +FROM golang:1.17.3 as builder ARG BUILD_VERSION=main ARG GIT_COMMIT=HEAD @@ -29,10 +28,7 @@ COPY pkg/ pkg/ RUN mkdir -p /apiserver.local.config/certificates && chmod -R 777 /apiserver.local.config # Build -# https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/ -ARG TARGETOS=linux -ARG TARGETARCH=amd64 -RUN VERSION=${BUILD_VERSION} GIT_COMMIT=${GIT_COMMIT} GIT_VERSION=${GIT_VERSION} TARGET_OS=$TARGETOS ARCH=$TARGETARCH make adapter +RUN VERSION=${BUILD_VERSION} GIT_COMMIT=${GIT_COMMIT} GIT_VERSION=${GIT_VERSION} make adapter # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details diff --git a/Makefile b/Makefile index 759ca007049..2b397a96648 100644 --- a/Makefile +++ b/Makefile @@ -69,18 +69,15 @@ all: build test: manifests generate fmt vet envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out -.PHONY: get-cluster-context -get-cluster-context: ## Get Azure cluster context. +.PHONY: e2e-test +e2e-test: ## Run e2e tests against Azure cluster. + TERMINFO=/etc/terminfo + TERM=linux @az login --service-principal -u $(AZURE_SP_ID) -p "$(AZURE_SP_KEY)" --tenant $(AZURE_SP_TENANT) @az aks get-credentials \ --name $(TEST_CLUSTER_NAME) \ --subscription $(AZURE_SUBSCRIPTION) \ --resource-group $(AZURE_RESOURCE_GROUP) - -.PHONY: e2e-test -e2e-test: get-cluster-context ## Run e2e tests against Azure cluster. - TERMINFO=/etc/terminfo - TERM=linux npm install --prefix tests ./tests/run-all.sh @@ -91,14 +88,9 @@ e2e-test-local: ## Run e2e tests against Kubernetes cluster configured in ~/.kub ./tests/run-all.sh .PHONY: e2e-test-clean -e2e-test-clean: get-cluster-context ## Delete all namespaces labeled with type=e2e +e2e-test-clean: ## Delete all namespaces labeled with type=e2e kubectl delete ns -l type=e2e -.PHONY: arm-smoke-test -arm-smoke-test: ## Run e2e tests against Kubernetes cluster configured in ~/.kube/config. - npm install --prefix tests - ./tests/run-arm-smoke-tests.sh - ################################################## # Development # ################################################## @@ -188,10 +180,6 @@ publish: docker-build ## Push images on to Container Registry (default: ghcr.io) docker push $(IMAGE_CONTROLLER) docker push $(IMAGE_ADAPTER) -publish-multiarch: - docker buildx build --push --platform=linux/amd64,linux/arm64 . -t ${IMAGE_CONTROLLER} --build-arg BUILD_VERSION=${VERSION} --build-arg GIT_VERSION=${GIT_VERSION} --build-arg GIT_COMMIT=${GIT_COMMIT} - docker buildx build --push --platform=linux/amd64,linux/arm64 -f Dockerfile.adapter -t ${IMAGE_ADAPTER} . --build-arg BUILD_VERSION=${VERSION} --build-arg GIT_VERSION=${GIT_VERSION} --build-arg GIT_COMMIT=${GIT_COMMIT} - release: manifests kustomize set-version ## Produce new KEDA release in keda-$(VERSION).yaml file. cd config/manager && \ $(KUSTOMIZE) edit set image ghcr.io/kedacore/keda=${IMAGE_CONTROLLER} @@ -289,10 +277,7 @@ endef help: ## Display this help. @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) -.PHONY: docker-build-tools -docker-build-tools: ## Build build-tools image - docker build -f tools/build-tools.Dockerfile -t $(IMAGE_BUILD_TOOLS) . - .PHONY: publish-build-tools -publish-build-tools: docker-build-tools ## Publish build-tools image +publish-build-tools: ## Publish build-tools image + docker build -f tools/build-tools.Dockerfile -t $(IMAGE_BUILD_TOOLS) . docker push $(IMAGE_BUILD_TOOLS) diff --git a/README.md b/README.md index 42ac5552618..50413c6e717 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,9 @@ We are a Cloud Native Computing Foundation (CNCF) incubation project. - [Getting started](#getting-started) - [Deploying KEDA](#deploying-keda) - [Documentation](#documentation) -- [Community](#community) -- [Adopters - Become a listed KEDA user!](#adopters---become-a-listed-keda-user) - [Governance & Policies](#governance--policies) -- [Roadmap](#roadmap) +- [Community](#community) + - [Become a listed KEDA user!](#become-a-listed-keda-user) - [Releases](#releases) - [Contributing](#contributing) - [Building & deploying locally](#building--deploying-locally) @@ -52,6 +51,10 @@ There are many ways to [deploy KEDA including Helm, Operator Hub and YAML files] Interested to learn more? Head over to [keda.sh](https://keda.sh). +## Governance & Policies + +You can learn about the governance of KEDA [here](https://github.com/kedacore/governance). + ## Community If interested in contributing or participating in the direction of KEDA, you can join our community meetings. @@ -59,24 +62,16 @@ If interested in contributing or participating in the direction of KEDA, you can * **Meeting time:** Bi-weekly Tues 15:00 UTC (does follow US daylight savings). ([Subscribe to Google Agenda](https://calendar.google.com/calendar?cid=bjE0bjJtNWM0MHVmam1ob2ExcTgwdXVkOThAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ) | [Convert to your timezone](https://www.thetimezoneconverter.com/?t=04%3A00%20pm&tz=UTC)) -* **Zoom link:** [https://zoom.us/j/96655859927?pwd=cGxaWWpHOVZSMEZDY3NuWWVIMERtdz09](https://zoom.us/j/96655859927?pwd=cGxaWWpHOVZSMEZDY3NuWWVIMERtdz09) (Password: keda) -* **Meeting agenda:** [Google Docs](https://docs.google.com/document/d/1zdwD6j86GxcCe5S5ay9suCO77WPrEDnKSfuaI24EwM4/edit?usp=sharing) +* **Zoom link:** [https://us02web.zoom.us/j/150360492?pwd=eUVtQzBPMzFoQUR2K1dqUWhENjJJdz09](https://us02web.zoom.us/j/150360492?pwd=eUVtQzBPMzFoQUR2K1dqUWhENjJJdz09) (Password: keda) +* **Meeting agenda:** [https://hackmd.io/s/r127ErYiN](https://hackmd.io/s/r127ErYiN) Just want to learn or chat about KEDA? Feel free to join the conversation in **[#KEDA](https://kubernetes.slack.com/messages/CKZJ36A5D)** on the **[Kubernetes Slack](https://slack.k8s.io/)**! -## Adopters - Become a listed KEDA user! +### Become a listed KEDA user! We are always happy to [list users](https://keda.sh/community/#users) who run KEDA in production, learn more about it [here](https://github.com/kedacore/keda-docs#become-a-listed-keda-user). -## Governance & Policies - -You can learn about the governance of KEDA [here](https://github.com/kedacore/governance). - -## Roadmap - -We use GitHub issues to build our backlog, a complete overview of all open items and our planning is available [here](https://github.com/orgs/kedacore/projects/2). - ## Releases You can find the latest releases [here](https://github.com/kedacore/keda/releases). diff --git a/SECURITY.md b/SECURITY.md index 4d83796c893..8b259ba83eb 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -14,19 +14,11 @@ Here's an overview: ## Prevention -We have a few preventive measures in place to detect security vulnerabilities: - -- [Renovate](https://renovatebot.com) helps us keep our dependencies up-to-date to patch vulnerabilities as soon as possible. -- [Snyk](https://snyk.io/) helps us ship secure container images: - - Images are scanned in every pull request (PR) to detect new vulnerabilities. - - Published images on GitHub Container Registry are monitored to detect new vulnerabilities so we can ship patches -- [Whitesource Bolt for GitHub](https://www.whitesourcesoftware.com/free-developer-tools/bolt/) helps us with identifying vulnerabilities in our dependencies to raise awareness. -- [GitHub's security features](https://github.com/features/security) are constantly monitoring our repo and dependencies: - - All pull requests (PRs) are using CodeQL to scan our source code for vulnerabilities - - Dependabot will automatically identify vulnerabilities based on GitHub Advisory Database and open PRs with patches +Container images are scanned in every pull request (PR) with [Snyk](https://snyk.io/) to detect new vulnerabilities. KEDA maintainers are working to improve our prevention by adding additional measures: +- Scan published container images on GitHub Container Registry ([issue](https://github.com/kedacore/keda/issues/1041)) - Scan changes to Helm charts in PRs ([issue](https://github.com/kedacore/charts/issues/64)) ## Disclosures diff --git a/pkg/scalers/azure_pipelines_scaler.go b/pkg/scalers/azure_pipelines_scaler.go index 8d0d8ec7ce8..df95f861bb9 100644 --- a/pkg/scalers/azure_pipelines_scaler.go +++ b/pkg/scalers/azure_pipelines_scaler.go @@ -41,6 +41,7 @@ type azurePipelinesMetadata struct { organizationURL string organizationName string personalAccessToken string + parent string poolID int targetPipelinesQueueLength int scalerIndex int @@ -100,6 +101,12 @@ func parseAzurePipelinesMetadata(ctx context.Context, config *ScalerConfig, http return nil, fmt.Errorf("no personalAccessToken given") } + if val, ok := config.TriggerMetadata["parent"]; ok && val != "" { + meta.parent = config.TriggerMetadata["parent"] + } else { + meta.parent = "" + } + if val, ok := config.TriggerMetadata["poolName"]; ok && val != "" { var err error meta.poolID, err = getPoolIDFromName(ctx, val, &meta, httpClient) @@ -230,13 +237,34 @@ func (s *azurePipelinesScaler) GetAzurePipelinesQueueLength(ctx context.Context) for _, value := range jobs { v := value.(map[string]interface{}) if v["result"] == nil { - count++ + if s.metadata.parent == "" { + // keep the old template working + count++ + } else if getCanAgentFulfilJob(v, s.metadata) { + count++ + } } } - return count, err } +// Determine if the Job and Agent have matching capabilities +func getCanAgentFulfilJob(v map[string]interface{}, metadata *azurePipelinesMetadata) bool { + matchedAgents, ok := v["matchedAgents"].([]interface{}) + if !ok { + // ADO is already processing + return false + } + + for _, m := range matchedAgents { + n := m.(map[string]interface{}) + if metadata.parent == n["name"].(string) { + return true + } + } + return false +} + func (s *azurePipelinesScaler) GetMetricSpecForScaling(context.Context) []v2beta2.MetricSpec { targetPipelinesQueueLengthQty := resource.NewQuantity(int64(s.metadata.targetPipelinesQueueLength), resource.DecimalSI) externalMetric := &v2beta2.ExternalMetricSource{ diff --git a/pkg/scalers/azure_pipelines_scaler_test.go b/pkg/scalers/azure_pipelines_scaler_test.go index be69a8b596a..db650ada0ad 100644 --- a/pkg/scalers/azure_pipelines_scaler_test.go +++ b/pkg/scalers/azure_pipelines_scaler_test.go @@ -2,6 +2,7 @@ package scalers import ( "context" + "fmt" "net/http" "net/http/httptest" "testing" @@ -158,3 +159,42 @@ func TestAzurePipelinesGetMetricSpecForScaling(t *testing.T) { } } } + +func getMatchedAgentMetaData(url string) *azurePipelinesMetadata { + meta := azurePipelinesMetadata{} + meta.organizationName = "test" + meta.organizationURL = url + meta.parent = "test-keda-template" + meta.personalAccessToken = "test" + meta.poolID = 1 + meta.targetPipelinesQueueLength = 1 + + return &meta +} + +func TestAzurePipelinesMatchedAgent(t *testing.T) { + var response = `{"count":1,"value":[{"demands":["Agent.Version -gtVersion 2.144.0"],"matchedAgents":[{"id":1,"name":"test-keda-template"}]}]}` + + var apiStub = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(response)) + })) + + meta := getMatchedAgentMetaData(apiStub.URL) + + mockAzurePipelinesScaler := azurePipelinesScaler{ + metadata: meta, + httpClient: http.DefaultClient, + } + + queuelen, err := mockAzurePipelinesScaler.GetAzurePipelinesQueueLength(context.TODO()) + + if err != nil { + t.Fail() + fmt.Println(err) + } + + if queuelen < 1 { + t.Fail() + } +} diff --git a/tests/run-arm-smoke-tests.sh b/tests/run-arm-smoke-tests.sh deleted file mode 100644 index 87e99fa137c..00000000000 --- a/tests/run-arm-smoke-tests.sh +++ /dev/null @@ -1,102 +0,0 @@ -#! /bin/bash -set -eu - -DIR=$(dirname "$0") -cd $DIR - -test_files=( - "influxdb.test.ts" #ScaledObject - "mongodb.test.ts" #ScaledJob -) - -concurrent_tests_limit=5 -pids=() -lookup=() -failed_count=0 -failed_lookup=() -counter=0 - -function run_setup { - ./node_modules/.bin/ava setup.test.ts -} - -function run_tests { - counter=0 - - for scaler in ${test_files[@]} - do - test_case="scalers/${scaler}" - counter=$((counter+1)) - ./node_modules/.bin/ava $test_case > "${test_case}.log" 2>&1 & - pid=$! - echo "Running $test_case with pid: $pid" - pids+=($pid) - lookup[$pid]=$test_case - # limit concurrent runs - if [[ "$counter" -gt "$concurrent_tests_limit" ]]; then - wait_for_jobs - counter=0 - pids=() - fi - done -} - -function mark_failed { - failed_lookup[$1]=${lookup[$1]} - let "failed_count+=1" -} - -function wait_for_jobs { - for job in "${pids[@]}"; do - wait $job || mark_failed $job - echo "Job $job finished" - done - - printf "\n$failed_count jobs failed\n" - printf '%s\n' "${failed_lookup[@]}" -} - -function print_logs { - for test_log in $(find scalers -name "*.log") - do - echo ">>> $test_log <<<" - cat $test_log - printf "\n\n##############################################\n" - printf "##############################################\n\n" - done - - echo ">>> KEDA Operator log <<<" - kubectl get pods --no-headers -n keda | awk '{print $1}' | grep keda-operator | xargs kubectl -n keda logs - printf "\n\n##############################################\n" - printf "##############################################\n\n" - - echo ">>> KEDA Metrics Server log <<<" - kubectl get pods --no-headers -n keda | awk '{print $1}' | grep keda-metrics-apiserver | xargs kubectl -n keda logs - printf "\n\n##############################################\n" - printf "##############################################\n\n" -} - -function run_cleanup { - ./node_modules/.bin/ava cleanup.test.ts -} - -function print_failed { - echo "$failed_count e2e tests failed" - for failed_test in "${failed_lookup[@]}"; do - echo $failed_test - done -} - -run_setup -run_tests -wait_for_jobs -print_logs -run_cleanup - -if [ "$failed_count" == "0" ]; -then - exit 0 -else - print_failed - exit 1 -fi diff --git a/tests/scalers/azure-blob.test.ts b/tests/scalers/azure-blob.test.ts index 24a6dc85994..995995e9477 100644 --- a/tests/scalers/azure-blob.test.ts +++ b/tests/scalers/azure-blob.test.ts @@ -4,26 +4,20 @@ import * as fs from 'fs' import * as sh from 'shelljs' import * as tmp from 'tmp' import test from 'ava' -import { createNamespace, waitForDeploymentReplicaCount } from './helpers' +import { createNamespace } from './helpers' const defaultNamespace = 'azure-blob-test' const connectionString = process.env['AZURE_STORAGE_CONNECTION_STRING'] // const blobSubPath = process.env['BLOB_SUB_PATH']; -test.before(async t => { +test.before(t => { if (!connectionString) { t.fail('AZURE_STORAGE_CONNECTION_STRING environment variable is required for blob tests') } // if (!blobSubPath) { // t.fail('BLOB_SUB_PATH environment variable is required for blob tests'); // } - const createContainerAsync = () => new Promise((resolve, _) => { - const blobSvc = azure.createBlobService(connectionString) - blobSvc.createContainerIfNotExists('container-name', err => { - resolve(undefined); - }) - }) - await createContainerAsync() + sh.config.silent = true const base64ConStr = Buffer.from(connectionString).toString('base64') @@ -41,13 +35,29 @@ test.serial('Deployment should have 0 replicas on start', t => { test.serial.cb('Deployment should scale to 2 with 150 blobs on the blob container then back to 0', t => { // add 2000 files const blobSvc = azure.createBlobService(connectionString) + blobSvc.createContainerIfNotExists('container-name', err => { + t.falsy(err, 'unable to create blob') + async.mapLimit(Array(350).keys(), 50, (n, cb) => blobSvc.createBlockBlobFromText('container-name',`blobsubpath/blob-name-${n}`,'test text', cb), () => { + let replicaCount = '0' + for (let i = 0; i < 40 && replicaCount !== '2'; i++) { + replicaCount = sh.exec(`kubectl get deployment.apps/test-deployment --namespace ${defaultNamespace} -o jsonpath="{.spec.replicas}"`).stdout + if (replicaCount !== '2') { + sh.exec('sleep 1s') + } + } - async.mapLimit(Array(350).keys(), 50, (n, cb) => blobSvc.createBlockBlobFromText('container-name',`blobsubpath/blob-name-${n}`,'test text', cb), async () => { - t.is(await waitForDeploymentReplicaCount(2,"test-deployment", defaultNamespace, 180, 1000),true, "Replica count should be 2 after 3 minute") + t.is('2', replicaCount, 'Replica count should be 2 after 40 seconds') - t.is(await waitForDeploymentReplicaCount(0,"test-deployment", defaultNamespace, 300, 1000),true, "Replica count should be 0 after 5 minute") + for (let i = 0; i < 50 && replicaCount !== '0'; i++) { + replicaCount = sh.exec(`kubectl get deployment.apps/test-deployment --namespace ${defaultNamespace} -o jsonpath="{.spec.replicas}"`).stdout + if (replicaCount !== '0') { + sh.exec('sleep 5s') + } + } - t.end() + t.is('0', replicaCount, 'Replica count should be 0 after 3 minutes') + t.end() + }) }) }) @@ -107,7 +117,7 @@ spec: secretKeyRef: name: test-secrets key: AzureWebJobsStorage - - name: TEST_STORAGE_CONNECTION_STRING + - name: AZURE_STORAGE_CONNECTION_STRING valueFrom: secretKeyRef: name: test-secrets @@ -130,5 +140,4 @@ spec: metadata: blobContainerName: container-name blobPrefix: blobsubpath - blobCount: '1' connectionFromEnv: AzureWebJobsStorage` diff --git a/tools/build-tools.Dockerfile b/tools/build-tools.Dockerfile index e3cc2e3d8a0..267c0afc7f7 100644 --- a/tools/build-tools.Dockerfile +++ b/tools/build-tools.Dockerfile @@ -23,12 +23,13 @@ RUN apt-get install apt-transport-https lsb-release dirmngr -y && \ apt-get update && \ apt-get install -y azure-cli -# Install docker -RUN apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common lsb-release && \ - curl -fsSL https://download.docker.com/linux/$(lsb_release -is | tr '[:upper:]' '[:lower:]')/gpg | apt-key add - 2>/dev/null && \ - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/$(lsb_release -is | tr '[:upper:]' '[:lower:]') $(lsb_release -cs) stable" && \ - apt-get update &&\ - apt-get install -y docker-ce-cli +# Install docker client +RUN curl -LO https://download.docker.com/linux/static/stable/x86_64/docker-19.03.2.tgz && \ + docker_sha256=865038730c79ab48dfed1365ee7627606405c037f46c9ae17c5ec1f487da1375 && \ + echo "$docker_sha256 docker-19.03.2.tgz" | sha256sum -c - && \ + tar xvzf docker-19.03.2.tgz && \ + mv docker/* /usr/local/bin && \ + rm -rf docker docker-19.03.2.tgz # Install golang RUN GO_VERSION=1.17.3 && \