From 0c838e4a724a22da850772d3d0d9f8a71622ef5d Mon Sep 17 00:00:00 2001 From: Priya Selvaganesan <1836417+priyaselvaganesan@users.noreply.github.com> Date: Tue, 29 Nov 2022 11:32:12 -0700 Subject: [PATCH] Add status metric to integration test for metric API verification (#289) --- Makefile | 2 +- hack/test/advanced-test.sh | 2 +- hack/test/run-e2e-tests.sh | 15 ++--- hack/test/test-wavefront-metrics.sh | 96 +++++++++++++++++++++++------ 4 files changed, 86 insertions(+), 29 deletions(-) diff --git a/Makefile b/Makefile index bdda1126..d246f796 100644 --- a/Makefile +++ b/Makefile @@ -252,7 +252,7 @@ undeploy: operator-yaml .PHONY: integration-test integration-test: install-kube-score install-kube-linter undeploy deploy - (cd $(REPO_DIR)/hack/test && ./run-e2e-tests.sh -t $(WAVEFRONT_TOKEN) -d $(NS) $(INTEGRATION_TEST_ARGS)) + (cd $(REPO_DIR)/hack/test && ./run-e2e-tests.sh -t $(WAVEFRONT_TOKEN) -d $(NS) -v $(VERSION) $(INTEGRATION_TEST_ARGS)) .PHONY: clean-cluster clean-cluster: diff --git a/hack/test/advanced-test.sh b/hack/test/advanced-test.sh index 2fffff8c..b174ac6f 100644 --- a/hack/test/advanced-test.sh +++ b/hack/test/advanced-test.sh @@ -1 +1 @@ -exit_on_fail wait_for_query_match_exact "ts(kubernetes.collector.version%2C%20cluster%3D%22${CONFIG_CLUSTER_NAME}%22%20AND%20installation_method%3D%22operator%22%20AND%20processed%3D%22true%22)" "${VERSION_IN_DECIMAL}" \ No newline at end of file +exit_on_fail wait_for_query_match_exact "ts(kubernetes.collector.version%2C%20cluster%3D%22${CONFIG_CLUSTER_NAME}%22%20AND%20installation_method%3D%22operator%22%20AND%20processed%3D%22true%22)" "${COLLECTOR_VERSION_IN_DECIMAL}" \ No newline at end of file diff --git a/hack/test/run-e2e-tests.sh b/hack/test/run-e2e-tests.sh index e17fbd33..c2d91a1c 100755 --- a/hack/test/run-e2e-tests.sh +++ b/hack/test/run-e2e-tests.sh @@ -7,10 +7,11 @@ NS=observability-system function print_usage_and_exit() { echo "Failure: $1" echo "Usage: $0 [flags] [options]" - echo -e "\t-c wavefront instance name (default: 'nimba')" echo -e "\t-t wavefront token (required)" + echo -e "\t-c wavefront instance name (default: 'nimba')" + echo -e "\t-v operator version (default: load from 'release/OPERATOR_VERSION')" echo -e "\t-n config cluster name for metric grouping (default: \$(whoami)--release-test)" - echo -e "\t-d namespace to create CR in" + echo -e "\t-d namespace to create CR in (default: observability-system" echo -e "\t-r tests to run (runs all by default)" exit 1 } @@ -39,7 +40,7 @@ function run_test_wavefront_metrics() { local cluster_name=${CONFIG_CLUSTER_NAME}-$type echo "Running test wavefront metrics, cluster_name $cluster_name ..." - ${REPO_ROOT}/hack/test/test-wavefront-metrics.sh -t ${WAVEFRONT_TOKEN} -n $cluster_name -v ${COLLECTOR_VERSION} -e "$type-test.sh" + ${REPO_ROOT}/hack/test/test-wavefront-metrics.sh -t ${WAVEFRONT_TOKEN} -n $cluster_name -e "$type-test.sh" -o ${VERSION} } function run_health_checks() { @@ -196,22 +197,22 @@ function main() { # REQUIRED local WAVEFRONT_TOKEN= + local WAVEFRONT_URL="https:\/\/nimba.wavefront.com" local WF_CLUSTER=nimba local VERSION=$(cat ${REPO_ROOT}/release/OPERATOR_VERSION) - local COLLECTOR_VERSION=$(cat ${REPO_ROOT}/release/COLLECTOR_VERSION) local K8S_ENV=$(cd ${REPO_ROOT}/hack/test && ./get-k8s-cluster-env.sh) local CONFIG_CLUSTER_NAME=$(create_cluster_name) local tests_to_run=() while getopts ":c:t:v:n:d:r:" opt; do case $opt in - c) - WF_CLUSTER="$OPTARG" - ;; t) WAVEFRONT_TOKEN="$OPTARG" ;; + c) + WF_CLUSTER="$OPTARG" + ;; v) VERSION="$OPTARG" ;; diff --git a/hack/test/test-wavefront-metrics.sh b/hack/test/test-wavefront-metrics.sh index 8272fd6b..4fb2ba85 100755 --- a/hack/test/test-wavefront-metrics.sh +++ b/hack/test/test-wavefront-metrics.sh @@ -14,17 +14,52 @@ function curl_query_to_wf_dashboard() { jq '.timeseries[0].data[0][1]' } +function wait_for_query_match_tags() { + local query=$1 + local expected_tags_json=$2 + local actual_tags_json=$(mktemp) + local loop_count=0 + + printf "Querying for tags %s ..." "$query" + + while [[ $loop_count -lt $MAX_QUERY_TIMES ]]; do + loop_count=$((loop_count + 1)) + END_TIME="$(date '+%s')000" + START_TIME="$(echo "`date +%s` - 120"| bc)000" + curl -s -X GET --header "Accept: application/json" \ + --header "Authorization: Bearer $WAVEFRONT_TOKEN" \ + "https://$WF_CLUSTER.wavefront.com/api/v2/chart/api?q=${query}&queryType=WQL&s=$START_TIME&e=$END_TIME&g=m&i=false&strict=true&view=METRIC&includeObsoleteMetrics=false&sorted=false&cached=true&useRawQK=false" | \ + jq -S '.timeseries[0].tags' | \ + sort | sed 's/,//g' > "$actual_tags_json" + printf "." + if [ "$(comm -23 "$expected_tags_json" "$actual_tags_json")" == "" ]; then + echo " done." + return 0 + fi + sleep $CURL_WAIT + done + + if [ "$(comm -23 "$expected_tags_json" "$actual_tags_json")" != "" ]; then + printf "\nChecking if expected tags are a subset of actual tags for query %s failed after attempting %s times.\n" "$query" "$MAX_QUERY_TIMES" + echo "Actual tags are:" + cat "$actual_tags_json" + echo "Expected tags are:" + cat "$expected_tags_json" + fi + return 1 +} + function wait_for_query_match_exact() { - local query_match_exact=$1 + local query=$1 local expected=$2 local actual local loop_count=0 - printf "Querying for exact match %s ..." "$query_match_exact" + printf "Querying for exact match %s ..." "$query" while [[ $loop_count -lt $MAX_QUERY_TIMES ]]; do loop_count=$((loop_count + 1)) - actual=$(curl_query_to_wf_dashboard "${query_match_exact}") + actual=$(curl_query_to_wf_dashboard "${query}") printf "." if echo "$actual $expected" | awk '{exit ($1 > $2 || $1 < $2)}'; then echo " done." @@ -34,6 +69,11 @@ function wait_for_query_match_exact() { sleep $CURL_WAIT done + if [[ $actual != $expected ]]; then + echo "Checking wavefront dashboard metrics for $query failed after attempting $MAX_QUERY_TIMES times." + echo "Actual is '$actual'" + echo "Expected is '$expected'" + fi return 1 } @@ -66,11 +106,13 @@ function wait_for_query_non_zero() { function print_usage_and_exit() { echo "Failure: $1" echo "Usage: $0 [flags] [options]" - echo -e "\t-c wavefront instance name (default: 'nimba')" echo -e "\t-t wavefront token (required)" - echo -e "\t-n config cluster name for metric grouping (default: \$(whoami)--release-test)" - echo -e "\t-v collector version (default: load from 'release/VERSION')" - echo -e "\t-e name of a file containing any extra asserts that should be made as part of this test" + echo -e "\t-n config cluster name for metric grouping (required)" + echo -e "\t-w wavefront instance name (default: 'nimba')" + echo -e "\t-c collector version (default: load from 'release/COLLECTOR_VERSION')" + echo -e "\t-o operator version (default: load from 'release/OPERATOR_VERSION')" + echo -e "\t-e name of a file containing any extra asserts that should be made as part of this test (optional)" + echo -e "\t-l name of test proxy used for logging (optional)" exit 1 } @@ -94,16 +136,18 @@ function main() { # REQUIRED local WAVEFRONT_TOKEN= + local CONFIG_CLUSTER_NAME= + local EXPECTED_COLLECTOR_VERSION=$(cat ${REPO_ROOT}/release/COLLECTOR_VERSION) + local EXPECTED_OPERATOR_VERSION=$(cat ${REPO_ROOT}/release/OPERATOR_VERSION) local WF_CLUSTER=nimba - local EXPECTED_VERSION=${COLLECTOR_VERSION} local EXTRA_TESTS= - local LOGGING_TEST_PROXY_NAME= - while getopts ":c:t:n:v:e:l:" opt; do + + while getopts ":c:t:n:o:c:e:l:" opt; do case $opt in - c) + w) WF_CLUSTER="$OPTARG" ;; t) @@ -112,8 +156,11 @@ function main() { n) CONFIG_CLUSTER_NAME="$OPTARG" ;; - v) - EXPECTED_VERSION="$OPTARG" + o) + EXPECTED_OPERATOR_VERSION="$OPTARG" + ;; + c) + EXPECTED_COLLECTOR_VERSION="$OPTARG" ;; e) EXTRA_TESTS="$OPTARG" @@ -128,20 +175,29 @@ function main() { done if [[ -z ${WAVEFRONT_TOKEN} ]]; then - print_msg_and_exit "wavefront token required" + print_usage_and_exit "wavefront token required" fi - if [[ -z ${CONFIG_CLUSTER_NAME} ]]; then - print_msg_and_exit "config cluster name required" + print_usage_and_exit "config cluster name required" fi - local VERSION_IN_DECIMAL="${EXPECTED_VERSION%.*}" - local VERSION_IN_DECIMAL+="$(echo "${EXPECTED_VERSION}" | cut -d '.' -f3)" - local VERSION_IN_DECIMAL="$(echo "${VERSION_IN_DECIMAL}" | sed 's/0$//')" + local COLLECTOR_VERSION_IN_DECIMAL="${EXPECTED_COLLECTOR_VERSION%.*}" + local COLLECTOR_VERSION_IN_DECIMAL+="$(echo "${EXPECTED_COLLECTOR_VERSION}" | cut -d '.' -f3)" + local COLLECTOR_VERSION_IN_DECIMAL="$(echo "${COLLECTOR_VERSION_IN_DECIMAL}" | sed 's/0$//')" wait_for_cluster_ready $NS - exit_on_fail wait_for_query_match_exact "ts(kubernetes.collector.version%2C%20cluster%3D%22${CONFIG_CLUSTER_NAME}%22%20AND%20installation_method%3D%22operator%22)" "${VERSION_IN_DECIMAL}" + local EXPECTED_TAGS_JSON=$(mktemp) + jq -S -n --arg status Healthy \ + --arg proxy Healthy \ + --arg metrics Healthy \ + --arg logging Healthy \ + --arg version "$EXPECTED_OPERATOR_VERSION" \ + '$ARGS.named' | \ + sort | sed 's/,//g' > "$EXPECTED_TAGS_JSON" + + exit_on_fail wait_for_query_match_tags "at(%22end%22%2C%202m%2C%20ts(%22kubernetes.observability.status%22%2C%20cluster%3D%22${CONFIG_CLUSTER_NAME}%22))" "${EXPECTED_TAGS_JSON}" + exit_on_fail wait_for_query_match_exact "ts(kubernetes.collector.version%2C%20cluster%3D%22${CONFIG_CLUSTER_NAME}%22%20AND%20installation_method%3D%22operator%22)" "${COLLECTOR_VERSION_IN_DECIMAL}" exit_on_fail wait_for_query_non_zero "ts(kubernetes.cluster.pod.count%2C%20cluster%3D%22${CONFIG_CLUSTER_NAME}%22)" if [[ ! -z ${LOGGING_TEST_PROXY_NAME} ]]; then