diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 602b330edf..0000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,25 +0,0 @@ - - -**Is this a BUG REPORT or FEATURE REQUEST?**: - -> Uncomment only one, leave it on its own line: -> -> /kind bug -> /kind feature - - -**What happened**: - -**What you expected to happen**: - -**How to reproduce it (as minimally and precisely as possible)**: - - -**Anything else we need to know?**: - -**Environment**: -- Kubernetes version (use `kubectl version`): -- Kube-state-metrics image version diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000000..79d1ddae87 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,27 @@ +--- +name: Bug report +about: Report a bug encountered while running kube-state-metrics +title: '' +labels: kind/bug +assignees: '' + +--- + + + +**What happened**: + +**What you expected to happen**: + +**How to reproduce it (as minimally and precisely as possible)**: + +**Anything else we need to know?**: + +**Environment**: +- kube-state-metrics version: +- Kubernetes version (use `kubectl version`): +- Cloud provider or hardware configuration: +- Other info: diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000000..f3fa65f79f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,18 @@ +--- +name: Feature request +about: Suggest a new feature +title: '' +labels: kind/feature +assignees: '' + +--- + + + +**What would you like to be added**: + +**Why is this needed**: + +**Describe the solution you'd like** + +**Additional context** diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index ae0e2b9883..9ee1c73c4e 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -7,6 +7,7 @@ **What this PR does / why we need it**: +**How does this change affect the cardinality of KSM**: *(increases, decreases or does not change cardinality)* + **Which issue(s) this PR fixes** *(optional, in `fixes #(, fixes #, ...)` format, will close the issue(s) when PR gets merged)*: Fixes # - diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..d9273f6418 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,209 @@ +name: continuous-integration + +on: + push: + branches: + - main + - release* + tags: + - v1.* + - v2.* + pull_request: + branches: + - main + - release* + +permissions: + contents: read + +env: + E2E_SETUP_KIND: yes + E2E_SETUP_KUBECTL: yes + SUDO: sudo + GO_VERSION: "^1.19" + GOLANGCI_LINT_VERSION: "v1.51.2" + +jobs: + ci-go-lint: + name: ci-go-lint + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: Setup environment + run: | + make install-tools + + - name: Lint + run: | + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin ${{ env.GOLANGCI_LINT_VERSION }} + make lint + + ci-validate-manifests: + name: ci-validate-manifests + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: Setup environment + run: | + make install-tools + + - name: Validate generated manifests + run: | + make validate-manifests + + ci-validate-go-modules: + name: ci-validate-go-modules + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: Setup environment + run: | + make install-tools + + - name: Validate go modules + run: | + make validate-modules + + ci-validate-docs: + name: ci-validate-docs + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: Setup environment + run: | + make install-tools + + - name: Check that all metrics are documented + run: | + make doccheck + + ci-unit-tests: + name: ci-unit-tests + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: Setup environment + run: | + make install-tools + + - name: Unit tests + run: | + make test-unit + + ci-rule-tests: + name: ci-rule-tests + runs-on: ubuntu-latest + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: Setup promtool + run: | + make install-promtool + + - name: Rule tests + run: | + PROMTOOL_CLI=./promtool make test-rules + + ci-benchmark-tests: + name: ci-benchmark-tests + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: Setup environment + run: | + make install-tools + + - name: Benchmark tests + run: | + make test-benchmark-compare + + ci-build-kube-state-metrics: + name: ci-build-kube-state-metrics + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: Setup environment + run: | + make install-tools + + - name: Build + run: | + make build + + ci-e2e-tests: + name: ci-e2e-tests + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: Setup environment + run: | + make install-tools + + - name: End-to-end tests + run: | + make e2e diff --git a/.github/workflows/govulncheck.yml b/.github/workflows/govulncheck.yml new file mode 100644 index 0000000000..2326a220e9 --- /dev/null +++ b/.github/workflows/govulncheck.yml @@ -0,0 +1,26 @@ +name: govulncheck + +on: + schedule: + # Run every Monday + - cron: '0 0 * * 1' + +env: + GO_VERSION: "^1.19" + +jobs: + ci-security-checks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + name: Checkout code + - name: Set up Go 1.x + uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: Install govulncheck binary + run: | + go install golang.org/x/vuln/cmd/govulncheck@latest + - name: Run security checks + run: | + govulncheck -v ./... diff --git a/.gitignore b/.gitignore index a40b46e952..33fd299d11 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,10 @@ documented_metrics code_metrics +# E2e tests output +/log +*.bak + # Created by https://www.gitignore.io/api/go ### Go ### @@ -9,6 +13,7 @@ code_metrics *.o *.a *.so +vendor # Folders _obj diff --git a/.golangci.yml b/.golangci.yml index f8f1849264..d7acc58f39 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,20 +1,38 @@ run: - deadline: 2m + deadline: 5m linters: disable-all: true enable: + - gocritic + - gocyclo - gofmt - goimports - - golint - - staticcheck - - gocyclo + - gosec + - gosimple + - govet - ineffassign - misspell - - gocritic - - govet + - promlinter + - revive + - staticcheck - unconvert linters-settings: goimports: - local-prefixes: k8s.io/kube-state-metrics + local-prefixes: k8s.io/kube-state-metrics,k8s.io/kube-state-metrics/v2 + +issues: + exclude-use-default: false + exclude-rules: + # We don't check metrics naming in the tests. + - path: _test\.go + linters: + - promlinter + # TODO(mrueg) Improve error handling + - text: "G104:" + linters: + - gosec + - text: "package-comments:" + linters: + - revive diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a8bee7b7f..1f12ececf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,291 @@ +## v2.8.2 / 2023-03-17 + +* [BUGFIX] Only use OpenMetrics and Text in contentType #2024 @CatherineF-dev + +## v2.8.1 / 2023-02-22 + +* [BUGFIX] Don't crash on non-existent paths @rexagod #1998 +* [BUGFIX] Fix public Builder compatibility with BuilderInterface @clamoriniere #1994 + +## v2.8.0 / 2023-02-10 + +Note: The `--version` flag was removed as `kube-state-metrics version` also provides the same information. See #1956 + +Note: Experimental CustomResourceState changed their labels for better usability. See #1942 +kube_crd_uptime{group="myteam.io", kind="Foo", version="v1"} -> kube_customresource_uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"} + +* [CHANGE] Prefix Group, Version and Kind labels for CustomResourceState Metrics #1942 @bavarianbidi +* [CHANGE] Fix empty string for "owner_\*" dimensions #1923 @pawcykca +* [CHANGE] Remove broken --version flag, replace by version command #1956 @bjorand +* [FEATURE] Add metrics for EndpointSlices #1910 @mrueg +* [FEATURE] Add metrics for config file changes #1916 @mrueg +* [FEATURE] Add metrics for CustomResource State config file change #1928 @mrueg +* [FEATURE] Reload Kube-State-Metrics on CustomResourceState config file change #1930 @mrueg +* [FEATURE] Make CustomResourceState metrics type dynamic #1930 @rexagod +* [FEATURE] Add kube_pod_status_qos_class to pod metrics #1932 @frezes +* [FEATURE] Add kube_pod_status_ready_time and kube_pod_status_containers_ready_time metrics #1938 @ryanrolds +* [FEATURE] Enrich UserAgent with more information about kube-state-metrics #1960 @mrueg +* [FEATURE] Convert True/False to 1.0/0.0 values in CustomResourceState metrics #1963 @jabdoa2 +* [FEATURE] Expose metrics in OpenMetrics format #1974 @mrueg +* [BUGFIX] Handle unit length `valueFrom` values #1958 @rexagod +* [ENHANCEMENT] Build with kubernetes 1.26 #1933 @mrueg + +## v2.7.0 / 2022-11-25 + +Note: Experimental VerticalPodAutoscaler metrics are considered deprecated in this release and will be removed in v2.9.0. +Please use CustomResourceStateMetrics to gather metrics from VPA resources. See: #1718 + +Note: Experimental CustomResourceState changed their naming convention for better usability. +The name of the CRD used to be interpolated into the name of the metric which made it impossible to aggregate a CRD across different versions. +This was changed to have the GVK information represented as labels: +kube_myteam_io_v1_Foo_uptime -> kube_crd_uptime{group="myteam.io", kind="Foo", version="v1"} +See: #1847 + +* [CHANGE] Deprecate VerticalPodAutoscaler metrics #1835 @rexagod +* [CHANGE] Recommend kube-scheduler metrics for container resource limits and requests #1849 @rexagod +* [FEATURE] Add experimental kube_pod_container_status_last_terminated_exitcode metric #1752 @ssabo +* [FEATURE] Introduce custom-resources-only flag #1813 @bavarianbidi +* [FEATURE] Allow allowlist filtering by wildcard key #1823 @rexagod +* [FEATURE] Add ContainerResourceSourceType to hpa spec and target metrics #1831 @whitebear009 +* [FEATURE] Represent group, version and kind of a resource as labels #1850 @rexagod +* [FEATURE] Sharding metrics per node via fieldSelector #1864 @CatherineF-dev +* [FEATURE] Add experimental StatefulSet retention policy metrics #1876 @mattcary +* [FEATURE] Allow labelFromKey field for all applicable metric types #1880 @rexagod +* [FEATURE] Introduce Viper, allow hot-reload on config change #1827 @rexagod +* [FEATURE] Introduce Cobra, allow configuration via environment variables #1834 @rexagod +* [FEATURE] Add experimental kube_node_deletion_timestamp metric #1890 @rexagod +* [FEATURE] Support autoscaling/v2 resources for HorizontalPodAutoscaler #1906 @JoaoBraveCoding +* [FEATURE] Add IngressClass metrics #1905 @kaitoii11 +* [ENHANCEMENT] Import Kubernetes metrics stability framework #1844 @CatherineF-dev +* [ENHANCEMENT] Promote kube_pod_container_status_waiting_reason and kube_deployment_status_replicas_ready to stable #1821 @CatherineF-dev +* [ENHANCEMENT] Build with Kubernetes 1.25 and go 1.19 #1819 @mrueg +* [BUGFIX] Handle singular labels in allowlist #1826 @rexagod +* [BUGFIX] Do not expose ingress path metric when service is nil #1841 @evir35 +* [BUGFIX] Allow lease metrics to be exported across all namespaces #1845 @lantingchiang + +## v2.6.0 / 2022-08-26 +* [FEATURE] Add local storage labels to kube_persistentvolume_info #1814 @nabokihms +* [FEATURE] Add support for StateSet and Info metrics for Custom-Resource State #1777 @chrischdi +* [FEATURE] Add support for Rolebinding resource metrics #1799 @kaitoii11 +* [FEATURE] Add new kube_horizontalpodautoscaler_status_target_metric #1725 @tanguyfalconnet +* [FEATURE] Add support for Role and Clusterrole resource metrics #1759 @kaitoii11 +* [FEATURE] Add support for the ServiceAccount resource metrics #1773 @Serializator +* [FEATURE] Add metric for pod tolerations #1769 @Serializator +* [FEATURE] Add new kube_endpoint_address metric #1761 @bavarianbidi +* [FEATURE] Support reading the timestamp fields of custom resources #1766 @dontan001 +* [ENHANCEMENT] Migrate to structured logging #1807 @dmpe +* [ENHANCEMENT] Switch registry vanity domain from k8s.gcr.io to registry.k8s.io #1750 @mrueg +* [ENHANCEMENT] Graduate new endpoint metrics to stable #1812 @bavarianbidi +* [BUGFIX] Fix label name for kube_pod_nodeselector #1756 @yosshi825 +* [BUGFIX] Fix Custom-Resource State Metrics not to contain underscores #1754 @chrischdi +* [BUGFIX] Fix kube_node_status_allocatable unit doc #1760 @jumbosushi + +## v2.5.0 / 2022-06-03 + +* [FEATURE] Add experimental Custom-Resource State Metrics #1710 @iamnoah +* [FEATURE] Add kube_pod_ips metric #1740 @bjschafer +* [FEATURE] Add kube_pod_nodeselector metric #1675 @geojaz +* [FEATURE] Add CSIPersistentVolumeSource to persistent volume metric #1727 @luke-sprig +* [FEATURE] Add kube_cronjob_status_last_successful_time metric #1732 @splitice +* [FEATURE] Add kube_persistentvolumeclaim_created metric #1741 @aidan-ogorman-dev +* [ENHANCEMENT] Build with Go 1.18 #1726, #1745 @mrueg +* [ENHANCEMENT] Bump kubernetes 1.24 and other go dependencies #1726, #1743 @mrueg +* [ENHANCEMENT] Update x/crypto to mitigate CVE-2022-27191 #1721 @pgvishnuram +* [BUGFIX] Assert that newlines in comma separated arguments are ignored #1706 @sthaha +* [BUGFIX] Fix potential panic in pod store #1723 @mfojtik +* [BUGFIX] Fix potential panic in internal store #1731 @jan--f +* [BUGFIX] Properly initialize KSM Server test #1699 @fpetkovski + +## v2.4.2 / 2022-02-10 +* [BUGFIX] Publish images with with the correct tag + +## v2.4.1 / 2022-02-10 +* [FEATURE] Add `ingressclass` label to `kube_ingress_info` metric #1652 @adammw +* [FEATURE] Extend KSM library to support custom resource metrics #1644 @Garrybest +* [ENHANCEMENT] Use apiVersion `v1` for `PodDisruptionBudget` and `CronJob` resources #1491 @bison +* [ENHANCEMENT] Optimize slice allocations #1676 @sherifabdlnaby +* [BUGFIX] Use plural form of resource name in `allowlist` for PodDisruptionBudget #1653 @arajkumar + +## v2.3.0 / 2021-12-09 + +* [FEATURE] Add a `--namespace-denylist` command line flag to exclude metrics from certain namespaces #1596 #1595 @mallow111 +* [FEATURE] Add `kube_*_labels` and `kube_*_annotations` metrics for Pod Disruption Budgets #1623 @arajkumar +* [FEATURE] Add a Kustomization file for deploying KSM in authosharding mode #1603 @karancode +* [FEATURE] Expose a metric for the number of ports in endpoint objects #1571 @bavarianbidi +* [FEATURE] Add a command line flag for opt-in metrics #1643 @Serializator +* [FEATURE] Add `kube_horizontalpodautoscaler_info` metric #1648 @Serializator +* [ENHANCEMENT] Update Go to 1.17.4 #1649 @fpetkovski +* [ENHANCEMENT] Update Kubernetes to 1.23 #1649 @fpetkovski +* [BUGFIX] Report the correct architecture for arm64 images #1629 @Serializator + +## v2.2.4 / 2021-11-08 + +* [BUGFIX] Fix BuilderInterface and BuildStoresFunc to allow using KSM as a library #1618 @ahmed-mez + +## v2.2.3 / 2021-10-13 + +* [BUGFIX] Fix the image build job. Reverts #1602 + +## v2.2.2 / 2021-10-13 +* [BUGFIX] Downgrade latest allowed go version to 1.16.9 #1601 @mrueg +* [BUGFIX] Fix CI variable names used for building KSM images @mrueg + +## v2.2.1 / 2021-09-24 + +* [FEATURE] Add the kube_persistentvolumeclaim_annotations metric which exposes annotations on PVCs #1566 @arajkumar +* [BUGFIX] Revert the accidentally removed kube_persistentvolumeclaim_labels metric #1566 @arajkumar +* [BUGFIX] Filter annotations in metrics based on `--metric-annotations-allowlist` instead of `--metric-labels-allowlist` for +CronJob, Daemonset, HPA and Ingress resources #1580 @midnw +* [BUGFIX] Avoid panicking when VPA objects have no targetRef #1584 @nabokihms + +## v2.2.0 / 2021-08-24 + +* [FEATURE] Add --use-apiserver-cache flag to set resourceVersion=0 for ListWatch requests #1548 +* [FEATURE] Introduce metrics for Kubernetes object annotations #1468 +* [FEATURE] Introduce start time metric for containers in terminated state #1519 +* [FEATURE] Introduce metrics for cronjob job history limits #1535 +* [FEATURE] Add system_uuid dimension to kube_node_info metric #1535 +* [FEATURE] Add available replica metric for statefulsets #1532 +* [FEATURE] Add ready replica metric for deployments #1534 +* [CHANGE] Update go clients for Kubernetes to support 1.22 #1545 +* [CHANGE] Use new promlint package and update prometheus cli to 2.28.1 #1531 + +## v2.1.1 / 2021-07-28 + +* [CHANGE] go.mod: Update and minimize dependencies #1529 +* [CHANGE] Use BuilderInterface instead of internal/store.Builder in metricshandler #1537 +* [CHANGE] Add WithAllowLabels to public BuilderInterface #1514 +* [BUGFIX] Fixes a bug where KSM did not export any metrics when it had no permissions for resources in at least one namespace #1499 + +## v2.1.0 / 2021-06-04 + +* [CHANGE] Update go version and dependencies #1493 + +## v2.1.0-rc.0 / 2021-05-20 + +* [FEATURE] Add support for native TLS #1354 +* [FEATURE] Add wildcard option to metric-labels-allowlist #1403 +* [FEATURE] Add build info metric #1332 +* [CHANGE] Add "uid" label to every pod metric #1304 +* [CHANGE] Add resourceVersion to CronJob metrics #1447 +* [CHANGE] Update go version and dependencies #1474 +* [CHANGE] Bump client-go and friends to v0.21 (Kubernetes v1.21) #1463 +* [CHANGE] Replace deprecated use of ioutil #1458 +* [BUGFIX] Fix builder.Builder WithMetrics signature #1455 +* [BUGFIX] Fix pod-metric missing reasons #1287 +* [BUGFIX] Fix multiListWatch resourceVersion mismatch if watch reconnected #1377 + +## v2.0.0 / 2021-04-13 + +* [CHANGE] Update go version and dependencies #1440 + +## v2.0.0-rc.1 / 2021-03-26 + +* [CHANGE] Rename --labels-metric-allow-list to --metric-labels-allowlist #1424 +* [CHANGE] Remove deprecated Kubernetes APIs #1423 +* [CHANGE] go.mod: Update Dependencies #1419 +* [CHANGE] Remove vendor folder #1419 +* [CHANGE] `k8s.gcr.io/kube-state-metrics/kube-state-metrics` becomes the authoritative registry +Location on quay.io will not be updated anymore. Previously pushed images will be kept around to avoid breaking existing deployments. + +## v2.0.0-rc.0 / 2021-03-04 + +* [CHANGE] internal/store/pod.go: Only create waiting_reason series if pods are in waiting state #1378 +* [CHANGE] internal/store/pod.go: Only create terminated_reason series if pods are in terminated state #1381 +* [CHANGE] internal/store/pod.go: Only create last_terminated containers series if containers are terminated state #1397 +* [FEATURE] Bump client-go and friends to v0.20 (kubernetes v1.20) #1328 +* [FEATURE] Bump go version to 1.16+ #1399 +* [BUGFIX] Fix gzip writer #1372 +* [BUGFIX] fix labels-metric-allow-list documentation #1404 +* [BUGFIX] Propagate resource version when sharded #1402 + +## v2.0.0-beta / 2020-12-04 +Promotion to beta release after a period of no bugs. + +## v2.0.0-alpha.3 / 2020-11-19 + +* [BUGFIX] Fix container resource limits metrics, which got dropped #1293 +* [BUGFIX] Adjust and refactor allowing labels to work for Kubernetes labels metrics #1301 +Note this is a breaking change, if you were using --labels-allow-list, look at the PR details for more information. + +## v2.0.0-alpha.2 / 2020-10-27 + +* [CHANGE] Migrate ingress and certificates to new stable APIs #1260 +* [CHANGE] Revert "Rework resource metrics" #1278 +To better align with future Kubernetes resource metrics, the changes to resource metrics were reverted, new metrics are: +kube_pod_container_resource_requests and kube_pod_container_resource_limits +* [FEATURE] Added the job failure reason in kube_job_status_failed metric #1214 +* [FEATURE] feat(persistentvolume): claimRef info to labels (kube_persistentvolume_claim_ref) #1244 +* [FEATURE] pod: add gauge for runtimeclass handler (kube_pod_runtimeclass_name_info) #1276 + +## v2.0.0-alpha.1 / 2020-10-06 + +* [CHANGE] Update go module path to k8s.io/kube-state-metrics/v2 #1238 +* [CHANGE] Bump klog to v2 and client-go to 1.19 #1250 +* [FEATURE] Add iscsi initiator name to persistentvolume_info #1235 +* [BUGFIX] Added Namespace to Rolebinding Jsonnet #1233 +* [BUGFIX] Reference closure scoped family generator #1240 + +## v2.0.0-alpha / 2020-09-16 + +NOTE: This is a major new alpha 2.0 release with breaking changes and removed metrics. See details below! + +* [CHANGE] Apply boundaries to metrics and allow via flag (--labels-allow-list) what labels to include #1125 +* [CHANGE] Update DaemonSet updated_number_scheduled metric name to be consistent #1181 +Metric was changed from kube_daemonset_updated_number_scheduled to kube_daemonset_status_updated_number_scheduled +* [CHANGE] Rework resource metrics #1168 +Metrics kube_pod_container_resource_requests, kube_pod_container_resource_limits, kube_pod_overhead, kube_pod_init_container_resource_limits, kube_pod_init_container_resource_requests changed +* [CHANGE] Convert k8s labels to snake case #1165 +* [CHANGE] Mutatingwebhookconfiguration.go: Switch to v1 #1144 +* [CHANGE] v2: Rename storage class labels reclaimPolicy to reclaim_policy and volumeBindingMode to volume_binding_mode #1107 +* [CHANGE] v2: Renamed --namespace flag to --namespaces #1098 +* [CHANGE] Rename kube_pod_deleted to kube_pod_deletion_timestamp #1079 +* [CHANGE] v2: Rename collector flag to resource flag #1006 +--resources is the new flag +* [CHANGE] Remove non-identifying labels from pod metrics #1009 +* [CHANGE] v2: Remove deprecated stable metrics #1004 +Note that some of these were replaced with EXPERIMENTAL resource metrics. See #1168 for more details. + `kube_pod_container_resource_requests` and `kube_pod_container_resource_limits` are the replacements with `resource` labels + representing the resource name and `unit` labels representing the resource unit. + - kube_pod_container_resource_requests_cpu_cores + - kube_pod_container_resource_limits_cpu_cores + - kube_pod_container_resource_requests_memory_bytes + - kube_pod_container_resource_limits_memory_bytes + - `kube_node_status_capacity` and `kube_node_status_allocatable` are the replacements with `resource` labels + representing the resource name and `unit` labels representing the resource unit. + - kube_node_status_capacity_pods + - kube_node_status_capacity_cpu_cores + - kube_node_status_capacity_memory_bytes + - kube_node_status_allocatable_pods + - kube_node_status_allocatable_cpu_cores + - kube_node_status_allocatable_memory_bytes +* [CHANGE] Rename black-/whitelist to allow/deny-list #1045 +New flags are --metric-allowlist and --metric-denylist +* [CHANGE] Update telemetry port to 8081 #1049 +* [CHANGE] v2: Rename hpa metrics to use full horizontalpodautoscaler #1003 +All metrics with prefix of kube_hpa_ were renamed to kube_horizontalpodautoscaler_ +* [CHANGE] v2: change metrics port to 8080, telemetry port to 8081 #1005 +* [FEATURE] Add http request metric for kube-state-metrics main /metrics #1218 +* [FEATURE] Add fc/iscsi/nfs identifier tags to persistentvolume_info metric #1208 +* [FEATURE] Adds new pod metric kube_pod_container_state_started #1183 +* [FEATURE] Add observedGeneration metric (kube_daemonset_status_observed_generation) for DaemonSets #1178 +* [FEATURE] Add internal_ip to node_info #1172 +* [FEATURE] Avoid conflicts when mapping Kubernetes labels to Prometheus labels #1156 +* [FEATURE] Add aws/gce volume id to kube_persistentvolume_info #1146 +* [FEATURE] Add UnexpectedAdmissionError to kube_pod_status_reason metric #1145 +* [FEATURE] Add init container requests (kube_pod_init_container_resource_requests) #1123 +* [FEATURE] Add host_network to kube_pod_info #1100 +* [FEATURE] Add kube_replicationcontroller_owner #1058 +* [FEATURE] Pod: add gauges for pod overhead (kube_pod_overhead) #1053 +* [FEATURE] Add "Terminating" status in kube_pod_status_phase metrics #1013 +* [FEATURE] Add lease collector metrics #1038 +* [ENHANCEMENT] Add DeprecatedVersion to struct FamilyGenerator and func NewFamilyGenerator #1160 +* [ENHANCEMENT] Add security context to deployment and statefulset #1034 +* [BUGFIX] Fix VolumeAttachment API version mismatch: expected v1 but watching v1beta1 #1136 +* [BUGFIX] Fix various CI issues (kube_volumeattachment_info nodeName -> node label rename) #1117 +* [BUGFIX] Fix maxUnavailable to round down instead up #1076 + + ## v1.9.7 / 2020-05-24 * [BUGFIX] internal/store/mutatingwebhookconfiguration.go: Switch to v1 #1144 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1337c231e8..914a8e2048 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,7 +14,7 @@ If your repo has certain guidelines for contribution, put them here ahead of the - [Contributor License Agreement](https://git.k8s.io/community/CLA.md) Kubernetes projects require that you sign a Contributor License Agreement (CLA) before we can accept your pull requests - [Kubernetes Contributor Guide](http://git.k8s.io/community/contributors/guide) - Main contributor documentation, or you can just jump directly to the [contributing section](http://git.k8s.io/community/contributors/guide#contributing) -- [Contributor Cheat Sheet](https://git.k8s.io/community/contributors/guide/contributor-cheatsheet.md) - Common resources for existing developers +- [Contributor Cheat Sheet](https://git.k8s.io/community/contributors/guide/contributor-cheatsheet/README.md) - Common resources for existing developers ## Mentorship diff --git a/Dockerfile b/Dockerfile index 7880f23598..c28baf6f1d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,15 @@ -FROM gcr.io/distroless/static +ARG GOVERSION=1.19 +ARG GOARCH +FROM golang:${GOVERSION} as builder +ARG GOARCH +ENV GOARCH=${GOARCH} +WORKDIR /go/src/k8s.io/kube-state-metrics/ +COPY . /go/src/k8s.io/kube-state-metrics/ -COPY kube-state-metrics / +RUN make build-local + +FROM gcr.io/distroless/static:latest-${GOARCH} +COPY --from=builder /go/src/k8s.io/kube-state-metrics/kube-state-metrics / USER nobody diff --git a/Dockerfile.prow b/Dockerfile.prow index 57e322ee02..8155feb1e9 100644 --- a/Dockerfile.prow +++ b/Dockerfile.prow @@ -1,6 +1,6 @@ # Copyright Contributors to the Open Cluster Management project -FROM registry.ci.openshift.org/stolostron/builder:go1.18-linux AS builder +FROM registry.ci.openshift.org/stolostron/builder:go1.19-linux AS builder WORKDIR /workspace COPY . . diff --git a/Makefile b/Makefile index 6892a1a128..06a3c843b5 100644 --- a/Makefile +++ b/Makefile @@ -1,33 +1,35 @@ FLAGS = TESTENVVAR = -REGISTRY = quay.io/coreos +REGISTRY ?= gcr.io/k8s-staging-kube-state-metrics TAG_PREFIX = v VERSION = $(shell cat VERSION) -TAG = $(TAG_PREFIX)$(VERSION) +TAG ?= $(TAG_PREFIX)$(VERSION) LATEST_RELEASE_BRANCH := release-$(shell grep -ohE "[0-9]+.[0-9]+" VERSION) +BRANCH = $(strip $(shell git rev-parse --abbrev-ref HEAD)) +DOCKER_CLI ?= docker +PROMTOOL_CLI ?= promtool PKGS = $(shell go list ./... | grep -v /vendor/ | grep -v /tests/e2e) ARCH ?= $(shell go env GOARCH) -BuildDate = $(shell date -u +'%Y-%m-%dT%H:%M:%SZ') -Commit = $(shell git rev-parse --short HEAD) +BUILD_DATE = $(shell date -u +'%Y-%m-%dT%H:%M:%SZ') +GIT_COMMIT ?= $(shell git rev-parse --short HEAD) +OS ?= $(shell uname -s | tr A-Z a-z) ALL_ARCH = amd64 arm arm64 ppc64le s390x -PKG = k8s.io/kube-state-metrics/pkg -GO_VERSION = 1.13 -FIRST_GOPATH := $(firstword $(subst :, ,$(shell go env GOPATH))) -BENCHCMP_BINARY := $(FIRST_GOPATH)/bin/benchcmp -GOLANGCI_VERSION := v1.19.1 -HAS_GOLANGCI := $(shell which golangci-lint) - +PKG = github.com/prometheus/common +PROMETHEUS_VERSION = 2.40.6 +GO_VERSION = 1.19.7 IMAGE = $(REGISTRY)/kube-state-metrics MULTI_ARCH_IMG = $(IMAGE)-$(ARCH) +USER ?= $(shell id -u -n) +HOST ?= $(shell hostname) + +export DOCKER_CLI_EXPERIMENTAL=enabled validate-modules: @echo "- Verifying that the dependencies have expected content..." go mod verify @echo "- Checking for any unused/missing packages in go.mod..." go mod tidy - @echo "- Checking for unused packages in vendor..." - go mod vendor - @git diff --exit-code -- go.sum go.mod vendor/ + @git diff --exit-code -- go.sum go.mod licensecheck: @echo ">> checking license header" @@ -40,17 +42,17 @@ licensecheck: fi lint: shellcheck licensecheck -ifndef HAS_GOLANGCI - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(GOPATH)/bin ${GOLANGCI_VERSION} -endif golangci-lint run +lint-fix: + golangci-lint run --fix -v + doccheck: generate @echo "- Checking if the generated documentation is up to date..." @git diff --exit-code @echo "- Checking if the documentation is in sync with the code..." - @grep -hoE '(kube_[^ |]+)' docs/* --exclude=README.md| sort -u > documented_metrics - @find internal/store -type f -not -name '*_test.go' -exec sed -nE 's/.*"(kube_[^"]+)"/\1/p' {} \; | sed -E 's/,//g' | sort -u > code_metrics + @grep -hoE -d skip '\| kube_[^ |]+' docs/* --exclude=README.md | sed -E 's/\| //g' | sort -u > documented_metrics + @find internal/store -type f -not -name '*_test.go' -exec sed -nE 's/.*"(kube_[^"]+)".*/\1/p' {} \; | sort -u > code_metrics @diff -u0 code_metrics documented_metrics || (echo "ERROR: Metrics with - are present in code but missing in documentation, metrics with + are documented but not found in code."; exit 1) @echo OK @rm -f code_metrics documented_metrics @@ -58,69 +60,58 @@ doccheck: generate @cd docs; for doc in *.md; do if [ "$$doc" != "README.md" ] && ! grep -q "$$doc" *.md; then echo "ERROR: No link to documentation file $${doc} detected"; exit 1; fi; done @echo OK -build-local: clean - GOOS=$(shell uname -s | tr A-Z a-z) GOARCH=$(ARCH) CGO_ENABLED=0 go build -ldflags "-s -w -X ${PKG}/version.Release=${TAG} -X ${PKG}/version.Commit=${Commit} -X ${PKG}/version.BuildDate=${BuildDate}" -o kube-state-metrics +build-local: + GOOS=$(OS) GOARCH=$(ARCH) CGO_ENABLED=0 go build -ldflags "-s -w -X ${PKG}/version.Version=${TAG} -X ${PKG}/version.Revision=${GIT_COMMIT} -X ${PKG}/version.Branch=${BRANCH} -X ${PKG}/version.BuildUser=${USER}@${HOST} -X ${PKG}/version.BuildDate=${BUILD_DATE}" -o kube-state-metrics + +build: kube-state-metrics -build: clean - docker run --rm -v "${PWD}:/go/src/k8s.io/kube-state-metrics" -w /go/src/k8s.io/kube-state-metrics golang:${GO_VERSION} make build-local +kube-state-metrics: + # Need to update git setting to prevent failing builds due to https://github.com/docker-library/golang/issues/452 + ${DOCKER_CLI} run --rm -v "${PWD}:/go/src/k8s.io/kube-state-metrics" -w /go/src/k8s.io/kube-state-metrics -e GOOS=$(OS) -e GOARCH=$(ARCH) golang:${GO_VERSION} git config --global --add safe.directory "*" && make build-local -test-unit: clean +test-unit: GOOS=$(shell uname -s | tr A-Z a-z) GOARCH=$(ARCH) $(TESTENVVAR) go test --race $(FLAGS) $(PKGS) +test-rules: + ${PROMTOOL_CLI} test rules tests/rules/alerts-test.yaml + shellcheck: - docker run -v "${PWD}:/mnt" koalaman/shellcheck:stable $(shell find . -type f -name "*.sh" -not -path "*vendor*") + ${DOCKER_CLI} run -v "${PWD}:/mnt" koalaman/shellcheck:stable $(shell find . -type f -name "*.sh" -not -path "*vendor*") # Runs benchmark tests on the current git ref and the last release and compares # the two. -test-benchmark-compare: $(BENCHCMP_BINARY) - ./tests/compare_benchmarks.sh master +test-benchmark-compare: + @git fetch + ./tests/compare_benchmarks.sh main ./tests/compare_benchmarks.sh ${LATEST_RELEASE_BRANCH} -TEMP_DIR := $(shell mktemp -d) - all: all-container +# Container build for multiple architectures as defined in ALL_ARCH + +container: container-$(ARCH) + +container-%: + ${DOCKER_CLI} build --pull -t $(IMAGE)-$*:$(TAG) --build-arg GOVERSION=$(GO_VERSION) --build-arg GOARCH=$* . + sub-container-%: $(MAKE) --no-print-directory ARCH=$* container -sub-push-%: - $(MAKE) --no-print-directory ARCH=$* push - all-container: $(addprefix sub-container-,$(ALL_ARCH)) -all-push: $(addprefix sub-push-,$(ALL_ARCH)) - -container: .container-$(ARCH) -.container-$(ARCH): - docker run --rm -v "${PWD}:/go/src/k8s.io/kube-state-metrics" -w /go/src/k8s.io/kube-state-metrics -e GOOS=linux -e GOARCH=$(ARCH) -e CGO_ENABLED=0 golang:${GO_VERSION} go build -ldflags "-s -w -X ${PKG}/version.Release=${TAG} -X ${PKG}/version.Commit=${Commit} -X ${PKG}/version.BuildDate=${BuildDate}" -o kube-state-metrics - cp -r * "${TEMP_DIR}" - docker build -t $(MULTI_ARCH_IMG):$(TAG) "${TEMP_DIR}" - docker tag $(MULTI_ARCH_IMG):$(TAG) $(MULTI_ARCH_IMG):latest - rm -rf "${TEMP_DIR}" - -ifeq ($(ARCH), amd64) - # Adding check for amd64 - docker tag $(MULTI_ARCH_IMG):$(TAG) $(IMAGE):$(TAG) - docker tag $(MULTI_ARCH_IMG):$(TAG) $(IMAGE):latest -endif - -quay-push: .quay-push-$(ARCH) -.quay-push-$(ARCH): .container-$(ARCH) - docker push $(MULTI_ARCH_IMG):$(TAG) - docker push $(MULTI_ARCH_IMG):latest -ifeq ($(ARCH), amd64) - docker push $(IMAGE):$(TAG) - docker push $(IMAGE):latest -endif - -push: .push-$(ARCH) -.push-$(ARCH): .container-$(ARCH) - gcloud docker -- push $(MULTI_ARCH_IMG):$(TAG) - gcloud docker -- push $(MULTI_ARCH_IMG):latest -ifeq ($(ARCH), amd64) - gcloud docker -- push $(IMAGE):$(TAG) - gcloud docker -- push $(IMAGE):latest -endif +# Container push, push is the target to push for multiple architectures as defined in ALL_ARCH + +push: $(addprefix sub-push-,$(ALL_ARCH)) push-multi-arch; + +sub-push-%: container-% do-push-% ; + +do-push-%: + ${DOCKER_CLI} push $(IMAGE)-$*:$(TAG) + +push-multi-arch: + ${DOCKER_CLI} manifest create --amend $(IMAGE):$(TAG) $(shell echo $(ALL_ARCH) | sed -e "s~[^ ]*~$(IMAGE)\-&:$(TAG)~g") + @for arch in $(ALL_ARCH); do ${DOCKER_CLI} manifest annotate --arch $${arch} $(IMAGE):$(TAG) $(IMAGE)-$${arch}:$(TAG); done + ${DOCKER_CLI} manifest push --purge $(IMAGE):$(TAG) clean: rm -f kube-state-metrics @@ -132,7 +123,7 @@ e2e: generate: build-local @echo ">> generating docs" @./scripts/generate-help-text.sh - @$(GOPATH)/bin/embedmd -w `find . -path ./vendor -prune -o -name "*.md" -print` + embedmd -w `find . -path ./vendor -prune -o -name "*.md" -print` validate-manifests: examples @git diff --exit-code @@ -160,6 +151,11 @@ scripts/vendor: scripts/jsonnetfile.json scripts/jsonnetfile.lock.json install-tools: @echo Installing tools from tools.go - @cat tools/tools.go | grep _ | awk -F'"' '{print $$2}' | xargs -tI % go install % + grep '^\s*_' tools/tools.go | awk '{print $$2}' | xargs -tI % go install -mod=readonly -modfile=tools/go.mod % + +install-promtool: + @echo Installing promtool + @wget -qO- "https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.${OS}-${ARCH}.tar.gz" |\ + tar xvz --strip-components=1 prometheus-${PROMETHEUS_VERSION}.${OS}-${ARCH}/promtool -.PHONY: all build build-local all-push all-container test-unit test-benchmark-compare container push quay-push clean e2e validate-modules shellcheck licensecheck lint generate embedmd +.PHONY: all build build-local all-push all-container container container-* do-push-* sub-push-* push push-multi-arch test-unit test-rules test-benchmark-compare clean e2e validate-modules shellcheck licensecheck lint lint-fix generate embedmd diff --git a/README.md b/README.md index caba3a6266..fba4fa72f3 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,11 @@ # Overview -[![Build Status](https://travis-ci.org/kubernetes/kube-state-metrics.svg?branch=master)](https://travis-ci.org/kubernetes/kube-state-metrics) [![Go Report Card](https://goreportcard.com/badge/github.com/kubernetes/kube-state-metrics)](https://goreportcard.com/report/github.com/kubernetes/kube-state-metrics) [![GoDoc](https://godoc.org/github.com/kubernetes/kube-state-metrics?status.svg)](https://godoc.org/github.com/kubernetes/kube-state-metrics) +[![Build Status](https://github.com/kubernetes/kube-state-metrics/workflows/continuous-integration/badge.svg)](https://github.com/kubernetes/kube-state-metrics/actions) +[![Go Report Card](https://goreportcard.com/badge/github.com/kubernetes/kube-state-metrics)](https://goreportcard.com/report/github.com/kubernetes/kube-state-metrics) +[![Go Reference](https://pkg.go.dev/badge/github.com/kubernetes/kube-state-metrics.svg)](https://pkg.go.dev/github.com/kubernetes/kube-state-metrics) +[![govulncheck](https://github.com/kubernetes/kube-state-metrics/actions/workflows/govulncheck.yml/badge.svg)](https://github.com/kubernetes/kube-state-metrics/actions/workflows/govulncheck.yml) -kube-state-metrics is a simple service that listens to the Kubernetes API +kube-state-metrics (KSM) is a simple service that listens to the Kubernetes API server and generates metrics about the state of the objects. (See examples in the Metrics section below.) It is not focused on the health of the individual Kubernetes components, but rather on the health of the various objects inside, @@ -18,35 +21,41 @@ Kubernetes API, this way users have all the data they require and perform heuristics as they see fit. The metrics are exported on the HTTP endpoint `/metrics` on the listening port -(default 80). They are served as plaintext. They are designed to be consumed +(default 8080). They are served as plaintext. They are designed to be consumed either by Prometheus itself or by a scraper that is compatible with scraping a Prometheus client endpoint. You can also open `/metrics` in a browser to see -the raw metrics. +the raw metrics. Note that the metrics exposed on the `/metrics` endpoint +reflect the current state of the Kubernetes cluster. When Kubernetes objects +are deleted they are no longer visible on the `/metrics` endpoint. ## Table of Contents -- [Overview](#overview) - - [Table of Contents](#table-of-contents) - - [Versioning](#versioning) - - [Kubernetes Version](#kubernetes-version) - - [Compatibility matrix](#compatibility-matrix) - - [Resource group version compatibility](#resource-group-version-compatibility) - - [Container Image](#container-image) - - [Metrics Documentation](#metrics-documentation) - - [Kube-state-metrics self metrics](#kube-state-metrics-self-metrics) - - [Scaling kube-state-metrics](#scaling-kube-state-metrics) - - [Resource recommendation](#resource-recommendation) - - [A note on costing](#a-note-on-costing) - - [kube-state-metrics vs. metrics-server](#kube-state-metrics-vs-metrics-server) - - [Horizontal scaling (sharding)](#horizontal-scaling-sharding) - - [Automated sharding](#automated-sharding) - - [Setup](#setup) - - [Building the Docker container](#building-the-docker-container) - - [Usage](#usage) - - [Kubernetes Deployment](#kubernetes-deployment) - - [Limited privileges environment](#limited-privileges-environment) - - [Development](#development) - - [Developer Contributions](#developer-contributions) +- [Versioning](#versioning) + - [Kubernetes Version](#kubernetes-version) + - [Compatibility matrix](#compatibility-matrix) + - [Resource group version compatibility](#resource-group-version-compatibility) + - [Container Image](#container-image) +- [Metrics Documentation](#metrics-documentation) + - [Conflict resolution in label names](#conflict-resolution-in-label-names) + - [Enabling VerticalPodAutoscalers](#enabling-verticalpodautoscalers) +- [Kube-state-metrics self metrics](#kube-state-metrics-self-metrics) +- [Resource recommendation](#resource-recommendation) +- [Latency](#latency) +- [A note on costing](#a-note-on-costing) +- [kube-state-metrics vs. metrics-server](#kube-state-metrics-vs-metrics-server) +- [Scaling kube-state-metrics](#scaling-kube-state-metrics) + - [Resource recommendation](#resource-recommendation) + - [Horizontal sharding](#horizontal-sharding) + - [Automated sharding](#automated-sharding) + - [Daemonset sharding for pod metrics](#daemonset-sharding-for-pod-metrics) +- [Setup](#setup) + - [Building the Docker container](#building-the-docker-container) +- [Usage](#usage) + - [Kubernetes Deployment](#kubernetes-deployment) + - [Limited privileges environment](#limited-privileges-environment) + - [Helm Chart](#helm-chart) + - [Development](#development) + - [Developer Contributions](#developer-contributions) ### Versioning @@ -59,20 +68,22 @@ The compatibility matrix for client-go and Kubernetes cluster can be found All additional compatibility is only best effort, or happens to still/already be supported. #### Compatibility matrix + At most, 5 kube-state-metrics and 5 [kubernetes releases](https://github.com/kubernetes/kubernetes/releases) will be recorded below. +Generally, it is recommended to use the latest release of kube-state-metrics. If you run a very recent version of Kubernetes, you might want to use an unreleased version to have the full range of supported resources. If you run an older version of Kubernetes, you might need to run an older version in order to have full support for all resources. Be aware, that the maintainers will only support the latest release. Older versions might be supported by interested users of the community. + +| kube-state-metrics | Kubernetes client-go Version | +|--------------------|:----------------------------:| +| **v2.4.2** | v1.23 | +| **v2.5.0** | v1.24 | +| **v2.6.0** | v1.24 | +| **v2.7.0** | v1.25 | +| **v2.8.1** | v1.26 | +| **main** | v1.26 | -| kube-state-metrics | **Kubernetes 1.12** | **Kubernetes 1.13** | **Kubernetes 1.14** | **Kubernetes 1.15** | **Kubernetes 1.16** | -|--------------------|---------------------|---------------------|---------------------|----------------------|----------------------| -| **v1.5.0** | ✓ | - | - | - | - | -| **v1.6.0** | ✓ | ✓ | - | - | - | -| **v1.7.2** | ✓ | ✓ | ✓ | - | - | -| **v1.8.0** | ✓ | ✓ | ✓ | ✓ | - | -| **v1.9.7** | ✓ | ✓ | ✓ | ✓ | ✓ | -| **master** | ✓ | ✓ | ✓ | ✓ | ✓ | -- `✓` Fully supported version range. -- `-` The Kubernetes cluster has features the client-go library can't use (additional API objects, etc). #### Resource group version compatibility + Resources in Kubernetes can evolve, i.e., the group version for a resource may change from alpha to beta and finally GA in different Kubernetes versions. For now, kube-state-metrics will only use the oldest API available in the latest release. @@ -80,38 +91,48 @@ release. #### Container Image The latest container image can be found at: -* `quay.io/coreos/kube-state-metrics:v1.9.7` -* `k8s.gcr.io/kube-state-metrics:v1.9.7` - -**Note**: -The recommended docker registry for kube-state-metrics is `quay.io`. kube-state-metrics on -`gcr.io` is only maintained on best effort as it requires external help from Google employees. +* `registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.8.1` (arch: `amd64`, `arm`, `arm64`, `ppc64le` and `s390x`) ### Metrics Documentation -There are many more metrics we could report, but this first pass is focused on -those that could be used for actionable alerts. Please contribute PR's for -additional metrics! - -> WARNING: THESE METRIC/TAG NAMES ARE UNSTABLE AND MAY CHANGE IN A FUTURE RELEASE. -> For now, the following metrics and collectors -> -> **metrics** -> * `kube_pod_container_resource_requests_nvidia_gpu_devices` -> * `kube_pod_container_resource_limits_nvidia_gpu_devices` -> * `kube_node_status_capacity_nvidia_gpu_cards` -> * `kube_node_status_allocatable_nvidia_gpu_cards` -> -> are removed in kube-state-metrics v1.4.0. -> -> Any collectors and metrics based on alpha Kubernetes APIs are excluded from any stability guarantee, -> which may be changed at any given release. +Any resources and metrics based on alpha Kubernetes APIs are excluded from any stability guarantee, +which may be changed at any given release. See the [`docs`](docs) directory for more information on the exposed metrics. +#### Conflict resolution in label names + +The `*_labels` family of metrics exposes Kubernetes labels as Prometheus labels. +As [Kubernetes](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set) +is more liberal than +[Prometheus](https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels) +in terms of allowed characters in label names, +we automatically convert unsupported characters to underscores. +For example, `app.kubernetes.io/name` becomes `label_app_kubernetes_io_name`. + +This conversion can create conflicts when multiple Kubernetes labels like +`foo-bar` and `foo_bar` would be converted to the same Prometheus label `label_foo_bar`. + +Kube-state-metrics automatically adds a suffix `_conflictN` to resolve this conflict, +so it converts the above labels to +`label_foo_bar_conflict1` and `label_foo_bar_conflict2`. + +If you'd like to have more control over how this conflict is resolved, +you might want to consider addressing this issue on a different level of the stack, +e.g. by standardizing Kubernetes labels using an +[Admission Webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/) +that ensures that there are no possible conflicts. + +#### Enabling VerticalPodAutoscalers + +Please note that the collector for `verticalpodautoscalers` is **disabled** by default; Vertical Pod Autoscaler metrics will not be collected until the collector is enabled. This is because Vertical Pod Autoscalers are managed as custom resources. + +If you want to enable this collector, +the [instructions](./docs/verticalpodautoscaler-metrics.md#Configuration) are located in the [Vertical Pod Autoscaler Metrics](./docs/verticalpodautoscaler-metrics.md) documentation. + ### Kube-state-metrics self metrics -kube-state-metrics exposes its own general process metrics under `--telemetry-host` and `--telemetry-port` (default 81). +kube-state-metrics exposes its own general process metrics under `--telemetry-host` and `--telemetry-port` (default 8081). kube-state-metrics also exposes list and watch success and error metrics. These can be used to calculate the error rate of list or watch resources. If you encounter those errors in the metrics, it is most likely a configuration or permission issue, and the next thing to investigate would be looking @@ -124,28 +145,62 @@ kube_state_metrics_list_total{resource="*v1.Node",result="error"} 52 kube_state_metrics_watch_total{resource="*v1beta1.Ingress",result="success"} 1 ``` +kube-state-metrics also exposes some http request metrics, examples of those are: +``` +http_request_duration_seconds_bucket{handler="metrics",method="get",le="2.5"} 30 +http_request_duration_seconds_bucket{handler="metrics",method="get",le="5"} 30 +http_request_duration_seconds_bucket{handler="metrics",method="get",le="10"} 30 +http_request_duration_seconds_bucket{handler="metrics",method="get",le="+Inf"} 30 +http_request_duration_seconds_sum{handler="metrics",method="get"} 0.021113919999999998 +http_request_duration_seconds_count{handler="metrics",method="get"} 30 +``` + +kube-state-metrics also exposes build and configuration metrics: +``` +kube_state_metrics_build_info{branch="main",goversion="go1.15.3",revision="6c9d775d",version="v2.0.0-beta"} 1 +kube_state_metrics_shard_ordinal{shard_ordinal="0"} 0 +kube_state_metrics_total_shards 1 +``` + +`kube_state_metrics_build_info` is used to expose version and other build information. For more usage about the info pattern, +please check the blog post [here](https://www.robustperception.io/exposing-the-software-version-to-prometheus). +Sharding metrics expose `--shard` and `--total-shards` flags and can be used to validate +run-time configuration, see [`/examples/prometheus-alerting-rules`](./examples/prometheus-alerting-rules). + +kube-state-metrics also exposes metrics about it config file and the Custom Resource State config file: + +``` +kube_state_metrics_config_hash{filename="crs.yml",type="customresourceconfig"} 2.38272279311849e+14 +kube_state_metrics_config_hash{filename="config.yml",type="config"} 2.65285922340846e+14 +kube_state_metrics_last_config_reload_success_timestamp_seconds{filename="crs.yml",type="customresourceconfig"} 1.6704882592037103e+09 +kube_state_metrics_last_config_reload_success_timestamp_seconds{filename="config.yml",type="config"} 1.6704882592035313e+09 +kube_state_metrics_last_config_reload_successful{filename="crs.yml",type="customresourceconfig"} 1 +kube_state_metrics_last_config_reload_successful{filename="config.yml",type="config"} 1 +``` + ### Scaling kube-state-metrics #### Resource recommendation -> Note: These recommendations are based on scalability tests done over a year ago. They may differ significantly today. - Resource usage for kube-state-metrics changes with the Kubernetes objects (Pods/Nodes/Deployments/Secrets etc.) size of the cluster. To some extent, the Kubernetes objects in a cluster are in direct proportion to the node number of the cluster. -As a general rule, you should allocate +As a general rule, you should allocate: -* 200MiB memory +* 250MiB memory * 0.1 cores -For clusters of more than 100 nodes, allocate at least +Note that if CPU limits are set too low, kube-state-metrics' internal queues will not be able to be worked off quickly enough, resulting in increased memory consumption as the queue length grows. If you experience problems resulting from high memory allocation or CPU throttling, try increasing the CPU limits. -* 2MiB memory per node -* 0.001 cores per node +### Latency -These numbers are based on [scalability tests](https://github.com/kubernetes/kube-state-metrics/issues/124#issuecomment-318394185) at 30 pods per node. +In a 100 node cluster scaling test the latency numbers were as follows: -Note that if CPU limits are set too low, kube-state-metrics' internal queues will not be able to be worked off quickly enough, resulting in increased memory consumption as the queue length grows. If you experience problems resulting from high memory allocation, try increasing the CPU limits. +``` +"Perc50": 259615384 ns, +"Perc90": 475000000 ns, +"Perc99": 906666666 ns. +``` ### A note on costing @@ -157,41 +212,73 @@ The [metrics-server](https://github.com/kubernetes-incubator/metrics-server) is a project that has been inspired by [Heapster](https://github.com/kubernetes-retired/heapster) and is implemented to serve the goals of core metrics pipelines in [Kubernetes monitoring -architecture](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/monitoring_architecture.md). +architecture](https://github.com/kubernetes/design-proposals-archive/blob/main/instrumentation/monitoring_architecture.md). It is a cluster level component which periodically scrapes metrics from all -Kubernetes nodes served by Kubelet through Summary API. The metrics are +Kubernetes nodes served by Kubelet through Metrics API. The metrics are aggregated, stored in memory and served in [Metrics API format](https://git.k8s.io/metrics/pkg/apis/metrics/v1alpha1/types.go). The -metric-server stores the latest values only and is not responsible for +metrics-server stores the latest values only and is not responsible for forwarding metrics to third-party destinations. kube-state-metrics is focused on generating completely new metrics from Kubernetes' object state (e.g. metrics based on deployments, replica sets, etc.). It holds an entire snapshot of Kubernetes state in memory and continuously generates new metrics based off of it. And just like the -metric-server it too is not responsibile for exporting its metrics anywhere. +metrics-server it too is not responsible for exporting its metrics anywhere. Having kube-state-metrics as a separate project also enables access to these metrics from monitoring systems such as Prometheus. -#### Horizontal scaling (sharding) +### Horizontal sharding -In order to scale kube-state-metrics horizontally, some automated sharding capabilities have been implemented. It is configured with the following flags: +In order to shard kube-state-metrics horizontally, some automated sharding capabilities have been implemented. It is configured with the following flags: * `--shard` (zero indexed) * `--total-shards` -Sharding is done by taking an md5 sum of the Kubernetes Object's UID and performing a modulo operation on it, with the total number of shards. The configured shard decides whether the object is handled by the respective instance of kube-state-metrics or not. Note that this means all instances of kube-state-metrics even if sharded will have the network traffic and the resource consumption for unmarshaling objects for all objects, not just the ones it is responsible for. To optimize this further, the Kubernetes API would need to support sharded list/watch capabilities. Overall memory consumption should be 1/n th of each shard compared to an unsharded setup. Typically, kube-state-metrics needs to be memory and latency optimized in order for it to return its metrics rather quickly to Prometheus. +Sharding is done by taking an md5 sum of the Kubernetes Object's UID and performing a modulo operation on it with the total number of shards. Each shard decides whether the object is handled by the respective instance of kube-state-metrics or not. Note that this means all instances of kube-state-metrics, even if sharded, will have the network traffic and the resource consumption for unmarshaling objects for all objects, not just the ones they are responsible for. To optimize this further, the Kubernetes API would need to support sharded list/watch capabilities. In the optimal case, memory consumption for each shard will be 1/n compared to an unsharded setup. Typically, kube-state-metrics needs to be memory and latency optimized in order for it to return its metrics rather quickly to Prometheus. One way to reduce the latency between kube-state-metrics and the kube-apiserver is to run KSM with the `--use-apiserver-cache` flag. In addition to reducing the latency, this option will also lead to a reduction in the load on etcd. + +Sharding should be used carefully and additional monitoring should be set up in order to ensure that sharding is set up and functioning as expected (eg. instances for each shard out of the total shards are configured). + +#### Automated sharding + +Automatic sharding allows each shard to discover its nominal position when deployed in a StatefulSet which is useful for automatically configuring sharding. This is an experimental feature and may be broken or removed without notice. -Sharding should be used carefully, and additional monitoring should be set up in order to ensure that sharding is set up and functioning as expected (eg. instances for each shard out of the total shards are configured). +To enable automated sharding, kube-state-metrics must be run by a `StatefulSet` and the pod name and namespace must be handed to the kube-state-metrics process via the `--pod` and `--pod-namespace` flags. Example manifests demonstrating the autosharding functionality can be found in [`/examples/autosharding`](./examples/autosharding). -##### Automated sharding +This way of deploying shards is useful when you want to manage KSM shards through a single Kubernetes resource (a single `StatefulSet` in this case) instead of having one `Deployment` per shard. The advantage can be especially significant when deploying a high number of shards. -There is also an experimental feature, that allows kube-state-metrics to auto discover its nominal position if it is deployed in a StatefulSet, in order to automatically configure sharding. This is an experimental feature and may be broken or removed without notice. +The downside of using an auto-sharded setup comes from the rollout strategy supported by `StatefulSet`s. When managed by a `StatefulSet`, pods are replaced one at a time with each pod first getting terminated and then recreated. Besides such rollouts being slower, they will also lead to short downtime for each shard. If a Prometheus scrape happens during a rollout, it can miss some of the metrics exported by kube-state-metrics. -To enable automated sharding kube-state-metrics must be run by a `StatefulSet` and the pod names and namespace must be handed to the kube-state-metrics process via the `--pod` and `--pod-namespace` flags. +### Daemonset sharding for pod metrics + +For pod metrics, they can be sharded per node with the following flag: +* `--node` + +Each kube-state-metrics pod uses FieldSelector (spec.nodeName) to watch/list pod metrics only on the same node. + +A daemonset kube-state-metrics example: +``` +apiVersion: apps/v1 +kind: DaemonSet +spec: + template: + spec: + containers: + - image: registry.k8s.io/kube-state-metrics/kube-state-metrics:IMAGE_TAG + name: kube-state-metrics + args: + - --resource=pods + - --node=$(NODE_NAME) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName +``` -There are example manifests demonstrating the autosharding functionality in [`/examples/autosharding`](./examples/autosharding). +Other metrics can be sharded via [Horizontal sharding](#horizontal-sharding). ### Setup @@ -214,10 +301,15 @@ make container Simply build and run kube-state-metrics inside a Kubernetes pod which has a service account token that has read-only access to the Kubernetes cluster. +#### For users of prometheus-operator/kube-prometheus stack + +The ([`kube-prometheus`](https://github.com/prometheus-operator/kube-prometheus/)) stack installs kube-state-metrics as one of its [components](https://github.com/prometheus-operator/kube-prometheus#kube-prometheus); you do not need to install kube-state-metrics if you're using the kube-prometheus stack. + +If you want to revise the default configuration for kube-prometheus, for example to enable non-default metrics, have a look at [Customizing Kube-Prometheus](https://github.com/prometheus-operator/kube-prometheus#customizing-kube-prometheus). + #### Kubernetes Deployment -To deploy this project, you can simply run `kubectl apply -f examples/standard` and a -Kubernetes service and deployment will be created. (Note: Adjust the apiVersion of some resource if your kubernetes cluster's version is not 1.8+, check the yaml file for more information). +To deploy this project, you can simply run `kubectl apply -f examples/standard` and a Kubernetes service and deployment will be created. (Note: Adjust the apiVersion of some resource if your kubernetes cluster's version is not 1.8+, check the yaml file for more information). To have Prometheus discover kube-state-metrics instances it is advised to create a specific Prometheus scrape config for kube-state-metrics that picks up both metrics endpoints. Annotation based discovery is discouraged as only one of the endpoints would be able to be selected, plus kube-state-metrics in most cases has special authentication and authorization requirements as it essentially grants read access through the metrics endpoint to most information available to it. @@ -261,7 +353,7 @@ subjects: namespace: your-namespace-where-kube-state-metrics-will-deployed ``` -- then specify a set of namespaces (using the `--namespace` option) and a set of kubernetes objects (using the `--collectors`) that your serviceaccount has access to in the `kube-state-metrics` deployment configuration +- then specify a set of namespaces (using the `--namespaces` option) and a set of kubernetes objects (using the `--resources`) that your serviceaccount has access to in the `kube-state-metrics` deployment configuration ```yaml spec: @@ -270,12 +362,17 @@ spec: containers: - name: kube-state-metrics args: - - '--collectors=pods' - - '--namespace=project1' + - '--resources=pods' + - '--namespaces=project1' ``` For the full list of arguments available, see the documentation in [docs/cli-arguments.md](./docs/cli-arguments.md) + +#### Helm Chart + +Starting from the kube-state-metrics chart `v2.13.3` (kube-state-metrics image `v1.9.8`), the official [Helm chart](https://artifacthub.io/packages/helm/prometheus-community/kube-state-metrics/) is maintained in [prometheus-community/helm-charts](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics). Starting from kube-state-metrics chart `v3.0.0` only kube-state-metrics images of `v2.0.0 +` are supported. + #### Development When developing, test a metric dump against your local Kubernetes cluster by diff --git a/RELEASE.md b/RELEASE.md index 5af9851123..072da51edf 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -6,18 +6,17 @@ We use [Semantic Versioning](http://semver.org/). We maintain a separate branch for each minor release, named `release-.`, e.g. `release-1.1`, `release-2.0`. -The usual flow is to merge new features and changes into the master branch and to merge bug fixes into the latest release branch. Bug fixes are then merged into master from the latest release branch. The master branch should always contain all commits from the latest release branch. +The usual flow is to merge new features and changes into the main branch and to merge bug fixes into the latest release branch. Bug fixes are then merged into main from the latest release branch. The main branch should always contain all commits from the latest release branch. -If a bug fix got accidentally merged into master, cherry-pick commits have to be created in the latest release branch, which then have to be merged back into master. Try to avoid that situation. +If a bug fix got accidentally merged into main, cherry-pick commits have to be created in the latest release branch, which then have to be merged back into main. Try to avoid that situation. Maintaining the release branches for older minor releases happens on a best effort basis. ## Prepare your release * Bump the version in the `VERSION` file in the root of the repository. -* Run `make examples`, which will re-generate all example manifests to use the the new version. +* Run `make examples`, which will re-generate all example manifests to use the new version. * Make a PR to update: - * kube-state-metrics image tag for both `quay.io` and `staging-k8s.gcr.io`. * [Compatibility matrix](README.md#compatibility-matrix) * Changelog entry * Only include user relevant changes @@ -30,10 +29,13 @@ Maintaining the release branches for older minor releases happens on a best effo ``` * All lines should be full sentences * kube-state-metrics image tag used in Kubernetes deployment yaml config. -* Cut the new release branch, i.e., `release-1.2`, or merge/cherry-pick changes onto the minor release branch you intend to tag the release on -* Cut the new release tag, i.e., `v1.2.0-rc.0` -* Ping Googlers(@loburm/@piosz) to build and push newest image to `k8s.gcr.io` (or to `staging-k8s.gcr.io` in case of release candidates) -* Build and push newest image to `quay.io`(@brancz) +* Cut the new release branch, e.g. `release-1.2`, or merge/cherry-pick changes onto the minor release branch you intend to tag the release on +* Cut the new release tag, e.g. `v1.2.0-rc.0` +* Create a new **pre-release** on github +* New images are automatically built and pushed to `gcr.io/k8s-staging-kube-state-metrics/kube-state-metrics` +* Promote image by sending a PR to [kubernetes/k8s.io](https://github.com/kubernetes/k8s.io) repository. Follow the [example PR](https://github.com/kubernetes/k8s.io/pull/3798). Use [kpromo pr](https://github.com/kubernetes-sigs/promo-tools/blob/main/docs/promotion-pull-requests.md) to update the manifest files in this repository, e.g. `kpromo pr --fork=$YOURNAME -i --project=kube-state-metrics -t=v2.5.0` +* Create a PR to merge the changes of this release back into the main branch. +* Once the PR to promote the image is merged, mark the pre-release as a regular release. ## Stable release diff --git a/SECURITY_CONTACTS b/SECURITY_CONTACTS index 2234eefe2e..007b4c0181 100644 --- a/SECURITY_CONTACTS +++ b/SECURITY_CONTACTS @@ -10,7 +10,6 @@ # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE # INSTRUCTIONS AT https://kubernetes.io/security/ -brancz -andyxning -tariq1890 -LiliC +dgrisonnet +fpetkovski +mrueg diff --git a/VERSION b/VERSION index fee0a2788b..1817afea41 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.9.7 +2.8.2 diff --git a/cloudbuild.yaml b/cloudbuild.yaml new file mode 100644 index 0000000000..018848abb6 --- /dev/null +++ b/cloudbuild.yaml @@ -0,0 +1,12 @@ +# See https://cloud.google.com/cloud-build/docs/build-config +timeout: 1800s +options: + substitution_option: ALLOW_LOOSE +steps: + - name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20220609-2e4c91eb7e' + entrypoint: make + env: + - TAG=$_PULL_BASE_REF + - GIT_COMMIT=$_PULL_BASE_SHA + args: + - push diff --git a/docs/README.md b/docs/README.md index a2903a32d3..a6f6840f30 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,7 +7,6 @@ Any contribution to improving this documentation or adding sample usages will be ## Table of Contents - [Metrics Stages](#metrics-stages) -- [Metrics Deprecation](#metrics-deprecation) - [Exposed Metrics](#exposed-metrics) - [Join Metrics](#join-metrics) - [CLI arguments](#cli-arguments) @@ -22,30 +21,18 @@ Stages about metrics are grouped into three categories： | STABLE | Metrics which should have very few backwards-incompatible changes outside of major version updates. | | DEPRECATED | Metrics which will be removed once the deprecation timeline is met. | -## Metrics Deprecation - -- **The following non-generic resource metrics for pods are marked deprecated. They will be removed in kube-state-metrics v2.0.0.** - `kube_pod_container_resource_requests` and `kube_pod_container_resource_limits` are the replacements with `resource` labels - representing the resource name and `unit` labels representing the resource unit. - - kube_pod_container_resource_requests_cpu_cores - - kube_pod_container_resource_limits_cpu_cores - - kube_pod_container_resource_requests_memory_bytes - - kube_pod_container_resource_limits_memory_bytes -- **The following non-generic resource metrics for nodes are marked deprecated. They will be removed in kube-state-metrics v2.0.0.** - `kube_node_status_capacity` and `kube_node_status_allocatable` are the replacements with `resource` labels - representing the resource name and `unit` labels representing the resource unit. - - kube_node_status_capacity_pods - - kube_node_status_capacity_cpu_cores - - kube_node_status_capacity_memory_bytes - - kube_node_status_allocatable_pods - - kube_node_status_allocatable_cpu_cores - - kube_node_status_allocatable_memory_bytes +## Opt-in Metrics + +As of v2.3.0, kube-state-metrics supports additional opt-in metrics via the CLI flag `--metric-opt-in-list`. See the metric documentation to identify which metrics need to be specified. ## Exposed Metrics -Per group of metrics there is one file for each metrics. See each file for specific documentation about the exposed metrics: +Per group of metrics there is one file for each metrics. +See each file for specific documentation about the exposed metrics: + +### Default Resources -- [CertificateSigningRequest Metrics](certificatessigningrequest-metrics.md) +- [CertificateSigningRequest Metrics](certificatesigningrequest-metrics.md) - [ConfigMap Metrics](configmap-metrics.md) - [CronJob Metrics](cronjob-metrics.md) - [DaemonSet Metrics](daemonset-metrics.md) @@ -54,8 +41,9 @@ Per group of metrics there is one file for each metrics. See each file for speci - [Horizontal Pod Autoscaler Metrics](horizontalpodautoscaler-metrics.md) - [Ingress Metrics](ingress-metrics.md) - [Job Metrics](job-metrics.md) +- [Lease Metrics](lease-metrics.md) - [LimitRange Metrics](limitrange-metrics.md) -- [MutatingWebhookConfiguration Metrics](mutatingwebhookconfiguration.md) +- [MutatingWebhookConfiguration Metrics](mutatingwebhookconfiguration-metrics.md) - [Namespace Metrics](namespace-metrics.md) - [NetworkPolicy Metrics](networkpolicy-metrics.md) - [Node Metrics](node-metrics.md) @@ -70,29 +58,43 @@ Per group of metrics there is one file for each metrics. See each file for speci - [Service Metrics](service-metrics.md) - [StatefulSet Metrics](statefulset-metrics.md) - [StorageClass Metrics](storageclass-metrics.md) -- [ValidatingWebhookConfiguration Metrics](validatingwebhookconfiguration.md) -- [VerticalPodAutoscaler Metrics](verticalpodautoscaler-metrics.md) +- [ValidatingWebhookConfiguration Metrics](validatingwebhookconfiguration-metrics.md) - [VolumeAttachment Metrics](volumeattachment-metrics.md) +### Optional Resources + +- [ClusterRole Metrics](clusterrole-metrics.md) +- [ClusterRoleBinding Metrics](clusterrolebinding-metrics.md) +- [EndpointSlice Metrics](endpointslice-metrics.md) +- [IngressClass Metrics](ingressclass-metrics.md) +- [Role Metrics](role-metrics.md) +- [RoleBinding Metrics](rolebinding-metrics.md) +- [ServiceAccount Metrics](serviceaccount-metrics.md) +- [VerticalPodAutoscaler Metrics](verticalpodautoscaler-metrics.md) + ## Join Metrics When an additional, not provided by default label is needed, a [Prometheus matching operator](https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching) can be used to extend single metrics output. This example adds `label_release` to the set of default labels of the `kube_pod_status_ready` metric -and allows you select or group the metrics by helm release label: +and allows you select or group the metrics by Helm release label: ``` -kube_pod_status_ready * on (namespace, pod) group_left(label_release) kube_pod_labels +kube_pod_status_ready * on (namespace, pod) group_left(label_release) kube_pod_labels ``` Another useful example would be to query the memory usage of pods by its `phase`, such as `Running`: ``` -sum(kube_pod_container_resource_requests_memory_bytes) by (namespace, pod, node) - * on (pod) group_left() (sum(kube_pod_status_phase{phase="Running"}) by (pod, namespace) == 1) +sum(kube_pod_container_resource_requests{resource="memory"}) by (namespace, pod, node) + * on (namespace, pod) group_left() (sum(kube_pod_status_phase{phase="Running"}) by (pod, namespace) == 1) ``` +## Metrics from Custom Resources + +See [Custom Resource State Metrics](customresourcestate-metrics.md) for experimental support for custom resources. + ## CLI Arguments Additionally, options for `kube-state-metrics` can be passed when executing as a CLI, or in a kubernetes / openshift environment. More information can be found here: [CLI Arguments](cli-arguments.md) diff --git a/docs/certificatesigningrequest-metrics.md b/docs/certificatesigningrequest-metrics.md new file mode 100644 index 0000000000..ca52fae725 --- /dev/null +++ b/docs/certificatesigningrequest-metrics.md @@ -0,0 +1,9 @@ +# CertificateSigningRequest Metrics + +| Metric name| Metric type | Labels/tags | Status | +| ---------- | ----------- | ----------- | ----------- | +| kube_certificatesigningrequest_annotations | Gauge | `certificatesigningrequest`=<certificatesigningrequest-name>
`signer_name`=<certificatesigningrequest-signer-name>| EXPERIMENTAL | +| kube_certificatesigningrequest_created| Gauge | `certificatesigningrequest`=<certificatesigningrequest-name>
`signer_name`=<certificatesigningrequest-signer-name>| STABLE | +| kube_certificatesigningrequest_condition | Gauge | `certificatesigningrequest`=<certificatesigningrequest-name>
`signer_name`=<certificatesigningrequest-signer-name>
`condition`=<approved\|denied> | STABLE | +| kube_certificatesigningrequest_labels | Gauge | `certificatesigningrequest`=<certificatesigningrequest-name>
`signer_name`=<certificatesigningrequest-signer-name>| STABLE | +| kube_certificatesigningrequest_cert_length | Gauge | `certificatesigningrequest`=<certificatesigningrequest-name>
`signer_name`=<certificatesigningrequest-signer-name>| STABLE | diff --git a/docs/certificatessigningrequest-metrics.md b/docs/certificatessigningrequest-metrics.md deleted file mode 100644 index 52548ab535..0000000000 --- a/docs/certificatessigningrequest-metrics.md +++ /dev/null @@ -1,8 +0,0 @@ -# CertificateSigningRequest Metrics - -| Metric name| Metric type | Labels/tags | Status | -| ---------- | ----------- | ----------- | ----------- | -| kube_certificatesigningrequest_created| Gauge | `certificatesigningrequest`=<certificatesigningrequest-name>| STABLE | -| kube_certificatesigningrequest_condition | Gauge | `certificatesigningrequest`=<certificatesigningrequest-name>
`condition`=<approved\|denied> | STABLE | -| kube_certificatesigningrequest_labels | Gauge | `certificatesigningrequest`=<certificatesigningrequest-name>| STABLE | -| kube_certificatesigningrequest_cert_length | Gauge | `certificatesigningrequest`=<certificatesigningrequest-name>| STABLE | diff --git a/docs/cli-arguments.md b/docs/cli-arguments.md index eb8f95d91f..0b812011b7 100644 --- a/docs/cli-arguments.md +++ b/docs/cli-arguments.md @@ -24,36 +24,58 @@ spec: [embedmd]:# (../help.txt) ```txt $ kube-state-metrics -h -Usage of ./kube-state-metrics: - --add_dir_header If true, adds the file directory to the header - --alsologtostderr log to standard error as well as files - --apiserver string The URL of the apiserver to use as a master - --collectors string Comma-separated list of collectors to be enabled. Defaults to "certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,horizontalpodautoscalers,ingresses,jobs,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,volumeattachments" - --disable-node-non-generic-resource-metrics Disable node non generic resource request and limit metrics - --disable-pod-non-generic-resource-metrics Disable pod non generic resource request and limit metrics - --enable-gzip-encoding Gzip responses when requested by clients via 'Accept-Encoding: gzip' header. - -h, --help Print Help text - --host string Host to expose metrics on. (default "0.0.0.0") - --kubeconfig string Absolute path to the kubeconfig file - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory - --log_file string If non-empty, use this log file - --log_file_max_size uint Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800) - --logtostderr log to standard error instead of files (default true) - --metric-blacklist string Comma-separated list of metrics not to be enabled. This list comprises of exact metric names and/or regex patterns. The whitelist and blacklist are mutually exclusive. - --metric-whitelist string Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or regex patterns. The whitelist and blacklist are mutually exclusive. - --namespace string Comma-separated list of namespaces to be enabled. Defaults to "" - --pod string Name of the pod that contains the kube-state-metrics container. When set, it is expected that --pod and --pod-namespace are both set. Most likely this should be passed via the downward API. This is used for auto-detecting sharding. If set, this has preference over statically configured sharding. This is experimental, it may be removed without notice. - --pod-namespace string Name of the namespace of the pod specified by --pod. When set, it is expected that --pod and --pod-namespace are both set. Most likely this should be passed via the downward API. This is used for auto-detecting sharding. If set, this has preference over statically configured sharding. This is experimental, it may be removed without notice. - --port int Port to expose metrics on. (default 80) - --shard int32 The instances shard nominal (zero indexed) within the total number of shards. (default 0) - --skip_headers If true, avoid header prefixes in the log messages - --skip_log_headers If true, avoid headers when opening log files - --stderrthreshold severity logs at or above this threshold go to stderr (default 2) - --telemetry-host string Host to expose kube-state-metrics self metrics on. (default "0.0.0.0") - --telemetry-port int Port to expose kube-state-metrics self metrics on. (default 81) - --total-shards int The total number of shards. Sharding is disabled when total shards is set to 1. (default 1) - -v, --v Level number for the log level verbosity - --version kube-state-metrics build version information - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging +kube-state-metrics is a simple service that listens to the Kubernetes API server and generates metrics about the state of the objects. + +Usage: + kube-state-metrics [flags] + kube-state-metrics [command] + +Available Commands: + completion Generate completion script for kube-state-metrics. + help Help about any command + version Print version information. + +Flags: + --add_dir_header If true, adds the file directory to the header of the log messages + --alsologtostderr log to standard error as well as files (no effect when -logtostderr=true) + --apiserver string The URL of the apiserver to use as a master + --config string Path to the kube-state-metrics options config file + --custom-resource-state-config string Inline Custom Resource State Metrics config YAML (experimental) + --custom-resource-state-config-file string Path to a Custom Resource State Metrics config file (experimental) + --custom-resource-state-only Only provide Custom Resource State metrics (experimental) + --enable-gzip-encoding Gzip responses when requested by clients via 'Accept-Encoding: gzip' header. + -h, --help Print Help text + --host string Host to expose metrics on. (default "::") + --kubeconfig string Absolute path to the kubeconfig file + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory (no effect when -logtostderr=true) + --log_file string If non-empty, use this log file (no effect when -logtostderr=true) + --log_file_max_size uint Defines the maximum size a log file can grow to (no effect when -logtostderr=true). Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800) + --logtostderr log to standard error instead of files (default true) + --metric-allowlist string Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive. + --metric-annotations-allowlist string Comma-separated list of Kubernetes annotations keys that will be used in the resource' labels metric. By default the metric contains only name and namespace labels. To include additional annotations provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]'). + --metric-denylist string Comma-separated list of metrics not to be enabled. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive. + --metric-labels-allowlist string Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the metric contains only name and namespace labels. To include additional labels provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided as a key, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'. + --metric-opt-in-list string Comma-separated list of metrics which are opt-in and not enabled by default. This is in addition to the metric allow- and denylists + --namespaces string Comma-separated list of namespaces to be enabled. Defaults to "" + --namespaces-denylist string Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, only namespaces that are excluded in namespaces-denylist will be used. + --node string Name of the node that contains the kube-state-metrics pod. Most likely it should be passed via the downward API. This is used for daemonset sharding. Only available for resources (pod metrics) that support spec.nodeName fieldSelector. This is experimental. + --one_output If true, only write logs to their native severity level (vs also writing to each lower severity level; no effect when -logtostderr=true) + --pod string Name of the pod that contains the kube-state-metrics container. When set, it is expected that --pod and --pod-namespace are both set. Most likely this should be passed via the downward API. This is used for auto-detecting sharding. If set, this has preference over statically configured sharding. This is experimental, it may be removed without notice. + --pod-namespace string Name of the namespace of the pod specified by --pod. When set, it is expected that --pod and --pod-namespace are both set. Most likely this should be passed via the downward API. This is used for auto-detecting sharding. If set, this has preference over statically configured sharding. This is experimental, it may be removed without notice. + --port int Port to expose metrics on. (default 8080) + --resources string Comma-separated list of Resources to be enabled. Defaults to "certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,horizontalpodautoscalers,ingresses,jobs,leases,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,volumeattachments" + --shard int32 The instances shard nominal (zero indexed) within the total number of shards. (default 0) + --skip_headers If true, avoid header prefixes in the log messages + --skip_log_headers If true, avoid headers when opening log files (no effect when -logtostderr=true) + --stderrthreshold severity logs at or above this threshold go to stderr when writing to files and stderr (no effect when -logtostderr=true or -alsologtostderr=false) (default 2) + --telemetry-host string Host to expose kube-state-metrics self metrics on. (default "::") + --telemetry-port int Port to expose kube-state-metrics self metrics on. (default 8081) + --tls-config string Path to the TLS configuration file + --total-shards int The total number of shards. Sharding is disabled when total shards is set to 1. (default 1) + --use-apiserver-cache Sets resourceVersion=0 for ListWatch requests, using cached resources from the apiserver instead of an etcd quorum read. + -v, --v Level number for the log level verbosity + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + +Use "kube-state-metrics [command] --help" for more information about a command. ``` diff --git a/docs/clusterrole-metrics.md b/docs/clusterrole-metrics.md new file mode 100644 index 0000000000..c60aee035a --- /dev/null +++ b/docs/clusterrole-metrics.md @@ -0,0 +1,9 @@ +# ClusterRole Metrics + +| Metric name| Metric type | Labels/tags | Status | +| ---------- | ----------- | ----------- | ----------- | +| kube_clusterrole_annotations | Gauge | `clusterrole`=<clusterrole-name> | EXPERIMENTAL +| kube_clusterrole_labels | Gauge | `clusterrole`=<clusterrole-name> | EXPERIMENTAL +| kube_clusterrole_info | Gauge | `clusterrole`=<clusterrole-name> | EXPERIMENTAL | +| kube_clusterrole_created | Gauge | `clusterrole`=<clusterrole-name> | EXPERIMENTAL | +| kube_clusterrole_metadata_resource_version | Gauge | `clusterrole`=<clusterrole-name> | EXPERIMENTAL | diff --git a/docs/clusterrolebinding-metrics.md b/docs/clusterrolebinding-metrics.md new file mode 100644 index 0000000000..936ae63726 --- /dev/null +++ b/docs/clusterrolebinding-metrics.md @@ -0,0 +1,9 @@ +# ClusterRoleBinding Metrics + +| Metric name| Metric type | Labels/tags | Status | +| ---------- | ----------- | ----------- | ----------- | +| kube_clusterrolebinding_annotations | Gauge | `clusterrolebinding`=<clusterrolebinding-name> | EXPERIMENTAL +| kube_clusterrolebinding_labels | Gauge | `clusterrolebinding`=<clusterrolebinding-name> | EXPERIMENTAL +| kube_clusterrolebinding_info | Gauge | `clusterrolebinding`=<clusterrolebinding-name>
`roleref_kind`=<role-kind>
`roleref_name`=<role-name> | EXPERIMENTAL | +| kube_clusterrolebinding_created | Gauge | `clusterrolebinding`=<clusterrolebinding-name> | EXPERIMENTAL | +| kube_clusterrolebinding_metadata_resource_version | Gauge | `clusterrolebinding`=<clusterrolebinding-name> | EXPERIMENTAL | diff --git a/docs/configmap-metrics.md b/docs/configmap-metrics.md index a143e82657..6e14046746 100644 --- a/docs/configmap-metrics.md +++ b/docs/configmap-metrics.md @@ -2,6 +2,8 @@ | Metric name| Metric type | Labels/tags | Status | | ---------- | ----------- | ----------- | ----------- | +| kube_configmap_annotations | Gauge | `configmap`=<configmap-name>
`namespace`=<configmap-namespace>
`annotation_CONFIGMAP_ANNOTATION`=<CONFIGMAP_ANNOTATION> | EXPERIMENTAL +| kube_configmap_labels | Gauge | `configmap`=<configmap-name>
`namespace`=<configmap-namespace>
`label_CONFIGMAP_LABEL`=<CONFIGMAP_LABEL> | STABLE | kube_configmap_info | Gauge | `configmap`=<configmap-name>
`namespace`=<configmap-namespace> | STABLE | | kube_configmap_created | Gauge | `configmap`=<configmap-name>
`namespace`=<configmap-namespace> | STABLE | | kube_configmap_metadata_resource_version | Gauge | `configmap`=<configmap-name>
`namespace`=<configmap-namespace> | EXPERIMENTAL | diff --git a/docs/cronjob-metrics.md b/docs/cronjob-metrics.md index 2ae5d0ca3c..a501ea868c 100644 --- a/docs/cronjob-metrics.md +++ b/docs/cronjob-metrics.md @@ -2,11 +2,16 @@ | Metric name| Metric type | Labels/tags | Status | | ---------- | ----------- | ----------- | ----------- | +| kube_cronjob_annotations | Gauge | `cronjob`=<cronjob-name>
`namespace`=<cronjob-namespace>
`annotation_CRONJOB_ANNOTATION`=<CRONJOB_ANNOTATION> | EXPERIMENTAL | kube_cronjob_info | Gauge | `cronjob`=<cronjob-name>
`namespace`=<cronjob-namespace>
`schedule`=<schedule>
`concurrency_policy`=<concurrency-policy> | STABLE | kube_cronjob_labels | Gauge | `cronjob`=<cronjob-name>
`namespace`=<cronjob-namespace>
`label_CRONJOB_LABEL`=<CRONJOB_LABEL> | STABLE | kube_cronjob_created | Gauge | `cronjob`=<cronjob-name>
`namespace`=<cronjob-namespace> | STABLE | kube_cronjob_next_schedule_time | Gauge | `cronjob`=<cronjob-name>
`namespace`=<cronjob-namespace> | STABLE | kube_cronjob_status_active | Gauge | `cronjob`=<cronjob-name>
`namespace`=<cronjob-namespace> | STABLE | kube_cronjob_status_last_schedule_time | Gauge | `cronjob`=<cronjob-name>
`namespace`=<cronjob-namespace> | STABLE +| kube_cronjob_status_last_successful_time | Gauge | `cronjob`=<cronjob-name>
`namespace`=<cronjob-namespace> | EXPERIMENTAL | kube_cronjob_spec_suspend | Gauge | `cronjob`=<cronjob-name>
`namespace`=<cronjob-namespace> | STABLE | kube_cronjob_spec_starting_deadline_seconds | Gauge | `cronjob`=<cronjob-name>
`namespace`=<cronjob-namespace> | STABLE +| kube_cronjob_metadata_resource_version| Gauge | `cronjob`=<cronjob-name>
`namespace`=<cronjob-namespace> | STABLE +| kube_cronjob_spec_successful_job_history_limit | Gauge | `cronjob`=<cronjob-name>
`namespace`=<cronjob-namespace> | EXPERIMENTAL +| kube_cronjob_spec_failed_job_history_limit | Gauge | `cronjob`=<cronjob-name>
`namespace`=<cronjob-namespace> | EXPERIMENTAL diff --git a/docs/customresourcestate-metrics.md b/docs/customresourcestate-metrics.md new file mode 100644 index 0000000000..aa8b9342e4 --- /dev/null +++ b/docs/customresourcestate-metrics.md @@ -0,0 +1,442 @@ +# Custom Resource State Metrics + +This section describes how to add metrics based on the state of a custom resource without writing a custom resource +registry and running your own build of KSM. + +## Configuration + +A YAML configuration file described below is required to define your custom resources and the fields to turn into metrics. + +Two flags can be used: + +* `--custom-resource-state-config "inline yaml (see example)"` or +* `--custom-resource-state-config-file /path/to/config.yaml` + +If both flags are provided, the inline configuration will take precedence. +When multiple entries for the same resource exist, kube-state-metrics will exit with an error. +This includes configuration which refers to a different API version. + +In addition to specifying one of `--custom-resource-state-config*` flags, you should also add the custom resource *Kind*s in plural form to the list of exposed resources in the `--resources` flag. If you don't specify `--resources`, then all known custom resources configured in `--custom-resource-state-config*` and all available default kubernetes objects will be taken into account by kube-state-metrics. + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-state-metrics + namespace: kube-system +spec: + template: + spec: + containers: + - name: kube-state-metrics + args: + - --custom-resource-state-config + # in YAML files, | allows a multi-line string to be passed as a flag value + # see https://yaml-multiline.info + - | + spec: + resources: + - groupVersionKind: + group: myteam.io + version: "v1" + kind: Foo + metrics: + - name: active_count + help: "Count of active Foo" + each: + type: Gauge + ... + - --resources=certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,foos,horizontalpodautoscalers,ingresses,jobs,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,volumeattachments,verticalpodautoscalers +``` + +It's also possible to configure kube-state-metrics to run in a `custom-resource-mode` only. In addition to specifying one of `--custom-resource-state-config*` flags, you could set `--custom-resource-state-only` to `true`. +With this configuration only the known custom resources configured in `--custom-resource-state-config*` will be taken into account by kube-state-metrics. + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-state-metrics + namespace: kube-system +spec: + template: + spec: + containers: + - name: kube-state-metrics + args: + - --custom-resource-state-config + # in YAML files, | allows a multi-line string to be passed as a flag value + # see https://yaml-multiline.info + - | + spec: + resources: + - groupVersionKind: + group: myteam.io + version: "v1" + kind: Foo + metrics: + - name: active_count + help: "Count of active Foo" + each: + type: Gauge + ... + - --custom-resource-state-only=true +``` + +NOTE: The `customresource_group`, `customresource_version`, and `customresource_kind` common labels are reserved, and will be overwritten by the values from the `groupVersionKind` field. + +### Examples + +The examples in this section will use the following custom resource: + +```yaml +kind: Foo +apiVersion: myteam.io/vl +metadata: + annotations: + bar: baz + qux: quxx + labels: + foo: bar + name: foo +spec: + version: v1.2.3 + order: + - id: 1 + value: true + - id: 3 + value: false + replicas: 1 +status: + phase: Pending + active: + type-a: 1 + type-b: 3 + conditions: + - name: a + value: 45 + - name: b + value: 66 + sub: + type-a: + active: 1 + ready: 2 + type-b: + active: 3 + ready: 4 + uptime: 43.21 +``` + +#### Single Values + +The config: + +```yaml +kind: CustomResourceStateMetrics +spec: + resources: + - groupVersionKind: + group: myteam.io + kind: "Foo" + version: "v1" + metrics: + - name: "uptime" + help: "Foo uptime" + each: + type: Gauge + gauge: + path: [status, uptime] +``` + +Produces the metric: + +```prometheus +kube_customresource_uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"} 43.21 +``` + +#### Multiple Metrics/Kitchen Sink + +```yaml +kind: CustomResourceStateMetrics +spec: + resources: + - groupVersionKind: + group: myteam.io + kind: "Foo" + version: "v1" + # labels can be added to all metrics from a resource + commonLabels: + crd_type: "foo" + labelsFromPath: + name: [metadata, name] + metrics: + - name: "ready_count" + help: "Number Foo Bars ready" + each: + type: Gauge + gauge: + # targeting an object or array will produce a metric for each element + # labelsFromPath and value are relative to this path + path: [status, sub] + + # if path targets an object, the object key will be used as label value + # This is not supported for StateSet type as all values will be truthy, which is redundant. + labelFromKey: type + # label values can be resolved specific to this path + labelsFromPath: + active: [active] + # The actual field to use as metric value. Should be a number, boolean or RFC3339 timestamp string. + valueFrom: [ready] + commonLabels: + custom_metric: "yes" + labelsFromPath: + # whole objects may be copied into labels by prefixing with "*" + # *anything will be copied into labels, with the highest sorted * strings first + "*": [metadata, labels] + "**": [metadata, annotations] + + # or specific fields may be copied. these fields will always override values from *s + name: [metadata, name] + foo: [metadata, labels, foo] +``` + +Produces the following metrics: + +```prometheus +kube_customresource_ready_count{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", active="1",custom_metric="yes",foo="bar",name="foo",bar="baz",qux="quxx",type="type-a"} 2 +kube_customresource_ready_count{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", active="3",custom_metric="yes",foo="bar",name="foo",bar="baz",qux="quxx",type="type-b"} 4 +``` + +### Metric types + +The configuration supports three kind of metrics from the [OpenMetrics specification](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md). + +The metric type is specified by the `type` field and its specific configuration at the types specific struct. + +#### Gauge + +> Gauges are current measurements, such as bytes of memory currently used or the number of items in a queue. For gauges the absolute value is what is of interest to a user. [[0]](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#gauge) + +Example: + +```yaml +kind: CustomResourceStateMetrics +spec: + resources: + - groupVersionKind: + group: myteam.io + kind: "Foo" + version: "v1" + metrics: + - name: "uptime" + help: "Foo uptime" + each: + type: Gauge + gauge: + path: [status, uptime] +``` + +Produces the metric: + +```prometheus +kube_customresource_uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"} 43.21 +``` + +##### Type conversion and special handling + +Gauges produce values of type float64 but custom resources can be of all kinds of types. +Kube-state-metrics performs implicity type conversions for a lot of type. +Supported types are: + +* (u)int32/64, int, float32 and byte are cast to float64 +* `nil` is generally mapped to `0.0` if NilIsZero is `true`. Otherwise it yields an error +* for bool `true` is mapped to `1.0` and `false` is mapped to `0.0` +* for string the following logic applies + * `"true"` and `"yes"` are mapped to `1.0` and `"false"` and `"no"` are mapped to `0.0` (all case insensitive) + * RFC3339 times are parsed to float timestamp + * finally the string is parsed to float using https://pkg.go.dev/strconv#ParseFloat which should support all common number formats. If that fails an error is yielded + +##### Example for status conditions on Kubernetes Controllers + +```yaml +kind: CustomResourceStateMetrics +spec: + resources: + - groupVersionKind: + group: myteam.io + kind: "Foo" + version: "v1" + labelsFromPath: + name: + - metadata + - name + namespace: + - metadata + - namespace + metrics: + - name: "foo_status" + help: "status condition " + each: + type: Gauge + gauge: + path: [status, conditions] + labelsFromPath: + type: ["type"] + valueFrom: ["status"] +``` + +This will work for kubernetes controller CRs which expose status conditions according to the kubernetes api (https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Condition): + +```yaml +status: + conditions: + - lastTransitionTime: "2019-10-22T16:29:31Z" + status: "True" + type: Ready +``` + +kube_customresource_foo_status{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", type="Ready"} 1.0 + +#### StateSet + +> StateSets represent a series of related boolean values, also called a bitset. If ENUMs need to be encoded this MAY be done via StateSet. [[1]](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#stateset) + +```yaml +kind: CustomResourceStateMetrics +spec: + resources: + - groupVersionKind: + group: myteam.io + kind: "Foo" + version: "v1" + metrics: + - name: "status_phase" + help: "Foo status_phase" + each: + type: StateSet + stateSet: + labelName: phase + path: [status, phase] + list: [Pending, Bar, Baz] +``` + +Metrics of type `StateSet` will generate a metric for each value defined in `list` for each resource. +The value will be 1, if the value matches the one in list. + +Produces the metric: + +```prometheus +kube_customresource_status_phase{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", phase="Pending"} 1 +kube_customresource_status_phase{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", phase="Bar"} 0 +kube_customresource_status_phase{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", phase="Baz"} 0 +``` + +#### Info + +> Info metrics are used to expose textual information which SHOULD NOT change during process lifetime. Common examples are an application's version, revision control commit, and the version of a compiler. [[2]](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#info) + +Metrics of type `Info` will always have a value of 1. + +```yaml +kind: CustomResourceStateMetrics +spec: + resources: + - groupVersionKind: + group: myteam.io + kind: "Foo" + version: "v1" + metrics: + - name: "version" + help: "Foo version" + each: + type: Info + info: + labelsFromPath: + version: [spec, version] +``` + +Produces the metric: + +```prometheus +kube_customresource_version{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", version="v1.2.3"} 1 +``` + +### Naming + +The default metric names are prefixed to avoid collisions with other metrics. +By default, a metric prefix of `kube_` concatenated with your custom resource's group+version+kind is used. +You can override this behavior with the `metricNamePrefix` field. + +```yaml +kind: CustomResourceStateMetrics +spec: + resources: + - groupVersionKind: ... + metricNamePrefix: myteam_foos + metrics: + - name: uptime + ... +``` + +Produces: +```prometheus +myteam_foos_uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"} 43.21 +``` + +To omit namespace and/or subsystem altogether, set them to the empty string: + +```yaml +kind: CustomResourceStateMetrics +spec: + resources: + - groupVersionKind: ... + metricNamePrefix: "" + metrics: + - name: uptime + ... +``` + +Produces: +```prometheus +uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"} 43.21 +``` + +### Logging + +If a metric path is registered but not found on a custom resource, an error will be logged. For some resources, +this may produce a lot of noise. The error log [verbosity][vlog] for a metric or resource can be set with `errorLogV` on +the resource or metric: + +```yaml +kind: CustomResourceStateMetrics +spec: + resources: + - groupVersionKind: ... + errorLogV: 0 # 0 = default for errors + metrics: + - name: uptime + errorLogV: 10 # only log at high verbosity +``` + +[vlog]: https://github.com/go-logr/logr#why-v-levels + +### Path Syntax + +Paths are specified as a list of strings. Each string is a path segment, resolved dynamically against the data of the custom resource. +If any part of a path is missing, the result is nil. + +Examples: + +```yaml +# simple path lookup +[spec, replicas] # spec.replicas == 1 + +# indexing an array +[spec, order, "0", value] # spec.order[0].value = true + +# finding an element in a list by key=value +[status, conditions, "[name=a]", value] # status.conditions[0].value = 45 + +# if the value to be matched is a number or boolean, the value is compared as a number or boolean +[status, conditions, "[value=66]", name] # status.conditions[1].name = "b" +``` diff --git a/docs/daemonset-metrics.md b/docs/daemonset-metrics.md index d974d22f1b..9a49b21158 100644 --- a/docs/daemonset-metrics.md +++ b/docs/daemonset-metrics.md @@ -2,6 +2,7 @@ | Metric name| Metric type | Labels/tags | Status | | ---------- | ----------- | ----------- | ----------- | +| kube_daemonset_annotations | Gauge | `daemonset`=<daemonset-name>
`namespace`=<daemonset-namespace>
`annotation_DAEMONSET_ANNOTATION`=<DAEMONSET_ANNOTATION> | EXPERIMENTAL | | kube_daemonset_created | Gauge | `daemonset`=<daemonset-name>
`namespace`=<daemonset-namespace> | STABLE | | kube_daemonset_status_current_number_scheduled | Gauge | `daemonset`=<daemonset-name>
`namespace`=<daemonset-namespace> | STABLE | | kube_daemonset_status_desired_number_scheduled | Gauge | `daemonset`=<daemonset-name>
`namespace`=<daemonset-namespace> | STABLE | @@ -9,6 +10,7 @@ | kube_daemonset_status_number_misscheduled | Gauge | `daemonset`=<daemonset-name>
`namespace`=<daemonset-namespace> | STABLE | | kube_daemonset_status_number_ready | Gauge | `daemonset`=<daemonset-name>
`namespace`=<daemonset-namespace> | STABLE | | kube_daemonset_status_number_unavailable | Gauge | `daemonset`=<daemonset-name>
`namespace`=<daemonset-namespace> | STABLE | -| kube_daemonset_updated_number_scheduled | Gauge | `daemonset`=<daemonset-name>
`namespace`=<daemonset-namespace> | STABLE | +| kube_daemonset_status_observed_generation | Gauge | `daemonset`=<daemonset-name>
`namespace`=<daemonset-namespace> | STABLE | +| kube_daemonset_status_updated_number_scheduled | Gauge | `daemonset`=<daemonset-name>
`namespace`=<daemonset-namespace> | STABLE | | kube_daemonset_metadata_generation | Gauge | `daemonset`=<daemonset-name>
`namespace`=<daemonset-namespace> | STABLE | | kube_daemonset_labels | Gauge | `daemonset`=<daemonset-name>
`namespace`=<daemonset-namespace>
`label_DAEMONSET_LABEL`=<DAEMONSET_LABEL> | STABLE | diff --git a/docs/deployment-metrics.md b/docs/deployment-metrics.md index 3681b610ef..e84b193fdb 100644 --- a/docs/deployment-metrics.md +++ b/docs/deployment-metrics.md @@ -2,7 +2,9 @@ | Metric name| Metric type | Labels/tags | Status | | ---------- | ----------- | ----------- | ----------- | +| kube_deployment_annotations | Gauge | `deployment`=<deployment-name>
`namespace`=<deployment-namespace>
`annotation_DEPLOYMENT_ANNOTATION`=<DEPLOYMENT_ANNOTATION> | EXPERIMENTAL | | kube_deployment_status_replicas | Gauge | `deployment`=<deployment-name>
`namespace`=<deployment-namespace> | STABLE | +| kube_deployment_status_replicas_ready | Gauge | `deployment`=<deployment-name>
`namespace`=<deployment-namespace> | STABLE | | kube_deployment_status_replicas_available | Gauge | `deployment`=<deployment-name>
`namespace`=<deployment-namespace> | STABLE | | kube_deployment_status_replicas_unavailable | Gauge | `deployment`=<deployment-name>
`namespace`=<deployment-namespace> | STABLE | | kube_deployment_status_replicas_updated | Gauge | `deployment`=<deployment-name>
`namespace`=<deployment-namespace> | STABLE | @@ -13,5 +15,5 @@ | kube_deployment_spec_strategy_rollingupdate_max_unavailable | Gauge | `deployment`=<deployment-name>
`namespace`=<deployment-namespace> | STABLE | | kube_deployment_spec_strategy_rollingupdate_max_surge | Gauge | `deployment`=<deployment-name>
`namespace`=<deployment-namespace> | STABLE | | kube_deployment_metadata_generation | Gauge | `deployment`=<deployment-name>
`namespace`=<deployment-namespace> | STABLE | -| kube_deployment_labels | Gauge | `deployment`=<deployment-name>
`namespace`=<deployment-namespace> | STABLE | +| kube_deployment_labels | Gauge | `deployment`=<deployment-name>
`namespace`=<deployment-namespace>
`label_DEPLOYMENT_LABEL`=<DEPLOYMENT_LABEL> | STABLE | | kube_deployment_created | Gauge | `deployment`=<deployment-name>
`namespace`=<deployment-namespace> | STABLE | diff --git a/docs/developer/guide.md b/docs/developer/guide.md index b728f7e6a0..eb5a72e51b 100644 --- a/docs/developer/guide.md +++ b/docs/developer/guide.md @@ -6,16 +6,28 @@ Any contribution to improving this documentation will be appreciated. ## Table of Contents - [Add New Kubernetes Resource Metric Collector](#add-new-kubernetes-resource-metric-collector) +- [Add New Metrics](#add-new-metrics) ### Add New Kubernetes Resource Metric Collector The following steps are needed to introduce a new resource and its respective resource metrics. -- Reference your new resource(s) to the [docs/README.md](https://github.com/kubernetes/kube-state-metrics/blob/master/docs/README.md#exposed-metrics). -- Reference your new resource(s) in the [docs/cli-arguments.md](https://github.com/kubernetes/kube-state-metrics/blob/master/docs/cli-arguments.md#available-options) as part of the `--collectors` flag. -- Create a new `.md` in the [docs](https://github.com/kubernetes/kube-state-metrics/tree/master/docs) directory to provide documentation on the resource(s) and metrics you implemented. Follow the formatting of all other resources. -- Add the resource(s) you are representing to the [examples/standard/cluster-role.yaml](https://github.com/kubernetes/kube-state-metrics/blob/master/examples/standard/cluster-role.yaml) under the appropriate `apiGroup` using the `verbs`: `list` and `watch`. -- Reference and add build functions for the new resource(s) in [internal/store/builder.go](https://github.com/kubernetes/kube-state-metrics/blob/master/internal/store/builder.go). -- Reference the new resource in [pkg/options/collector.go](https://github.com/kubernetes/kube-state-metrics/blob/master/pkg/options/collector.go). -- Add a sample Kubernetes manifest to be used by tests in the [tests/manifests/](https://github.com/kubernetes/kube-state-metrics/tree/master/tests/manifests) directory. -- Lastly, and most importantly, actually implement your new resource(s) and its test binary in [internal/store](https://github.com/kubernetes/kube-state-metrics/tree/master/internal/store). Follow the formatting and structure of other resources. +- Reference your new resource(s) to the [docs/README.md](https://github.com/kubernetes/kube-state-metrics/blob/main/docs/README.md#exposed-metrics). +- Reference your new resource(s) in the [docs/cli-arguments.md](https://github.com/kubernetes/kube-state-metrics/blob/main/docs/cli-arguments.md#available-options) as part of the `--resources` flag. +- Create a new `.md` in the [docs](https://github.com/kubernetes/kube-state-metrics/tree/main/docs) directory to provide documentation on the resource(s) and metrics you implemented. Follow the formatting of all other resources. +- Add the resource(s) you are representing to the [jsonnet/kube-state-metrics/kube-state-metrics.libsonnet](https://github.com/kubernetes/kube-state-metrics/blob/main/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet) under the appropriate `apiGroup` using the `verbs`: `list` and `watch`. +- Run `make examples/standard`, this should re-generate [examples/standard/cluster-role.yaml](https://github.com/kubernetes/kube-state-metrics/blob/main/examples/standard/cluster-role.yaml) with the resource(s) added to [jsonnet/kube-state-metrics/kube-state-metrics.libsonnet](https://github.com/kubernetes/kube-state-metrics/blob/main/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet). +- Reference and add build functions for the new resource(s) in [internal/store/builder.go](https://github.com/kubernetes/kube-state-metrics/blob/main/internal/store/builder.go). +- Reference the new resource in [pkg/options/resource.go](https://github.com/kubernetes/kube-state-metrics/blob/main/pkg/options/resource.go). +- Add a sample Kubernetes manifest to be used by tests in the [tests/manifests/](https://github.com/kubernetes/kube-state-metrics/tree/main/tests/manifests) directory. +- Lastly, and most importantly, actually implement your new resource(s) and its test binary in [internal/store](https://github.com/kubernetes/kube-state-metrics/tree/main/internal/store). Follow the formatting and structure of other resources. + +### Add New Metrics + +- Make metrics experimental first when introducing them, refer [#1910](https://github.com/kubernetes/kube-state-metrics/pull/1910) for more information. + +| Metric stability level | | +|------------------------|--------------------| +| EXPERIMENTAL | basemetrics.ALPHA | +| STABLE | basemetrics.STABLE | + diff --git a/docs/endpoint-metrics.md b/docs/endpoint-metrics.md index 2e0b416030..281772b20d 100644 --- a/docs/endpoint-metrics.md +++ b/docs/endpoint-metrics.md @@ -2,8 +2,11 @@ | Metric name| Metric type | Labels/tags | Status | | ---------- | ----------- | ----------- | ----------- | -| kube_endpoint_address_not_ready | Gauge | `endpoint`=<endpoint-name>
`namespace`=<endpoint-namespace> | STABLE | -| kube_endpoint_address_available | Gauge | `endpoint`=<endpoint-name>
`namespace`=<endpoint-namespace> | STABLE | +| kube_endpoint_annotations | Gauge | `endpoint`=<endpoint-name>
`namespace`=<endpoint-namespace>
`annotation_ENDPOINT_ANNOTATION`=<ENDPOINT_ANNOTATION> | EXPERIMENTAL | +| kube_endpoint_address_not_ready | Gauge | `endpoint`=<endpoint-name>
`namespace`=<endpoint-namespace> | DEPRECATED | +| kube_endpoint_address_available | Gauge | `endpoint`=<endpoint-name>
`namespace`=<endpoint-namespace> | DEPRECATED | | kube_endpoint_info | Gauge | `endpoint`=<endpoint-name>
`namespace`=<endpoint-namespace> | STABLE | | kube_endpoint_labels | Gauge | `endpoint`=<endpoint-name>
`namespace`=<endpoint-namespace>
`label_ENDPOINT_LABEL`=<ENDPOINT_LABEL> | STABLE | | kube_endpoint_created | Gauge | `endpoint`=<endpoint-name>
`namespace`=<endpoint-namespace> | STABLE | +| kube_endpoint_ports | Gauge | `endpoint`=<endpoint-name>
`namespace`=<endpoint-namespace>
`port_name`=<endpoint-port-name>
`port_protocol`=<endpoint-port-protocol>
`port_number`=<endpoint-port-number> | STABLE | +| kube_endpoint_address | Gauge | `endpoint`=<endpoint-name>
`namespace`=<endpoint-namespace>
`ip`=<endpoint-ip>
`ready`=<true if available, false if unavailalbe> | STABLE | diff --git a/docs/endpointslice-metrics.md b/docs/endpointslice-metrics.md new file mode 100644 index 0000000000..f2327a5265 --- /dev/null +++ b/docs/endpointslice-metrics.md @@ -0,0 +1,10 @@ +# Endpoint Metrics + +| Metric name| Metric type | Labels/tags | Status | +| ---------- | ----------- | ----------- | ----------- | +| kube_endpointslice_annotations | Gauge | `endpointslice`=<endpointslice-name>
`namespace`=<endpointslice-namespace>
`annotation_ENDPOINTSLICE_ANNOTATION`=<ENDPOINTSLICE_ANNOTATION> | EXPERIMENTAL | +| kube_endpointslice_info | Gauge | `endpointslice`=<endpointslice-name>
`namespace`=<endpointslice-namespace> | EXPERIMENTAL | +| kube_endpointslice_ports | Gauge | `endpointslice`=<endpointslice-name>
`namespace`=<endpointslice-namespace>
`port_name`=<endpointslice-port-name>
`port_protocol`=<endpointslice-port-protocol>
`port_number`=<endpointslice-port-number> | EXPERIMENTAL | +| kube_endpointslice_endpoints | Gauge | `endpointslice`=<endpointslice-name>
`namespace`=<endpointslice-namespace>
`ready`=<endpointslice-ready>
`serving`=<endpointslice-serving>
`terminating`=<endpointslice-terminating>
`hostname`=<endpointslice-hostname>
`targetref_kind`=<endpointslice-targetref-kind>
`targetref_name`=<endpointslice-targetref-name>
`targetref_namespace`=<endpointslice-targetref-namespace>
`nodename`=<endpointslice-nodename>
`endpoint_zone`=<endpointslice-zone> | EXPERIMENTAL | +| kube_endpointslice_labels | Gauge | `endpointslice`=<endpointslice-name>
`namespace`=<endpointslice-namespace>
`label_ENDPOINTSLICE_LABEL`=<ENDPOINTSLICE_LABEL> | EXPERIMENTAL | +| kube_endpointslice_created | Gauge | `endpointslice`=<endpointslice-name>
`namespace`=<endpointslice-namespace> | EXPERIMENTAL | diff --git a/docs/horizontalpodautoscaler-metrics.md b/docs/horizontalpodautoscaler-metrics.md index 72ff98e66b..2c909acdbf 100644 --- a/docs/horizontalpodautoscaler-metrics.md +++ b/docs/horizontalpodautoscaler-metrics.md @@ -2,11 +2,14 @@ | Metric name | Metric type | Labels/tags | Status | | -------------------------------- | ----------- | ------------------------------------------------------------- | ------ | -| kube_hpa_labels | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | -| kube_hpa_metadata_generation | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | -| kube_hpa_spec_max_replicas | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | -| kube_hpa_spec_min_replicas | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | -| kube_hpa_spec_target_metric | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace>
`metric_name`=<metric-name>
`metric_target_type`=<value\|utilization\|average> | EXPERIMENTAL | -| kube_hpa_status_condition | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace>
`condition`=<hpa-condition>
`status`=<true\|false\|unknown> | STABLE | -| kube_hpa_status_current_replicas | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | -| kube_hpa_status_desired_replicas | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | +| kube_horizontalpodautoscaler_info | Gauge | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace>
`scaletargetref_api_version`=<hpa-target-api-version>
`scaletargetref_kind`=<hpa-target-kind>
`scaletargetref_name`=<hpa-target-name> | EXPERIMENTAL | +| kube_horizontalpodautoscaler_annotations | Gauge | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace> | EXPERIMENTAL | +| kube_horizontalpodautoscaler_labels | Gauge | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | +| kube_horizontalpodautoscaler_metadata_generation | Gauge | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | +| kube_horizontalpodautoscaler_spec_max_replicas | Gauge | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | +| kube_horizontalpodautoscaler_spec_min_replicas | Gauge | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | +| kube_horizontalpodautoscaler_spec_target_metric | Gauge | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace>
`metric_name`=<metric-name>
`metric_target_type`=<value\|utilization\|average> | EXPERIMENTAL | +| kube_horizontalpodautoscaler_status_target_metric | Gauge | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace>
`metric_name`=<metric-name>
`metric_target_type`=<value\|utilization\|average> | EXPERIMENTAL | +| kube_horizontalpodautoscaler_status_condition | Gauge | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace>
`condition`=<hpa-condition>
`status`=<true\|false\|unknown> | STABLE | +| kube_horizontalpodautoscaler_status_current_replicas | Gauge | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | +| kube_horizontalpodautoscaler_status_desired_replicas | Gauge | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | diff --git a/docs/ingress-metrics.md b/docs/ingress-metrics.md index c435b2da4e..960d663026 100644 --- a/docs/ingress-metrics.md +++ b/docs/ingress-metrics.md @@ -2,9 +2,10 @@ | Metric name| Metric type | Labels/tags | Status | | ---------- | ----------- | ----------- | ----------- | -| kube_ingress_info | Gauge | `ingress`=<ingress-name>
`namespace`=<ingress-namespace> | STABLE | +| kube_ingress_annotations | Gauge | `ingress`=<ingress-name>
`namespace`=<ingress-namespace>
`annotation_INGRESS_ANNOTATION`=<ANNOTATION_LABEL> | EXPERIMENTAL | +| kube_ingress_info | Gauge | `ingress`=<ingress-name>
`namespace`=<ingress-namespace>
`ingressclass`=<ingress-class> or `_default` if not set | STABLE | | kube_ingress_labels | Gauge | `ingress`=<ingress-name>
`namespace`=<ingress-namespace>
`label_INGRESS_LABEL`=<INGRESS_LABEL> | STABLE | | kube_ingress_created | Gauge | `ingress`=<ingress-name>
`namespace`=<ingress-namespace> | STABLE | | kube_ingress_metadata_resource_version | Gauge | `ingress`=<ingress-name>
`namespace`=<ingress-namespace> | EXPERIMENTAL | -| kube_ingress_path | Gauge | `ingress`=<ingress-name>
`namespace`=<ingress-namespace>
`host`=<ingress-host>
`path`=<ingress-path>
`service_name`=<service name for the path>
`service_port`=<service port for hte path> | STABLE | +| kube_ingress_path | Gauge | `ingress`=<ingress-name>
`namespace`=<ingress-namespace>
`host`=<ingress-host>
`path`=<ingress-path>
`service_name`=<service name for the path>
`service_port`=<service port for the path> | STABLE | | kube_ingress_tls | Gauge | `ingress`=<ingress-name>
`namespace`=<ingress-namespace>
`tls_host`=<tls hostname>
`secret`=<tls secret name>| STABLE | diff --git a/docs/ingressclass-metrics.md b/docs/ingressclass-metrics.md new file mode 100644 index 0000000000..f9fade6171 --- /dev/null +++ b/docs/ingressclass-metrics.md @@ -0,0 +1,8 @@ +# IngressClass Metrics + +| Metric name| Metric type | Labels/tags | Status | +| ---------- | ----------- | ----------- | ----------- | +| kube_ingressclass_annotations | Gauge | `ingressclass`=<ingressclass-name>
`annotation_INGRESSCLASS_ANNOTATION`=<INGRESSCLASS_ANNOTATION> | EXPERIMENTAL | +| kube_ingressclass_info | Gauge | `ingressclass`=<ingressclass-name>
`controller`=<ingress-controller-name>
| EXPERIMENTAL | +| kube_ingressclass_labels | Gauge | `ingressclass`=<ingressclass-name>
`label_INGRESSCLASS_LABEL`=<INGRESSCLASS_LABEL> | EXPERIMENTAL| +| kube_ingressclass_created | Gauge | `ingressclass`=<ingressclass-name> | EXPERIMENTAL| diff --git a/docs/job-metrics.md b/docs/job-metrics.md index 385cd4a11f..6d82a882d3 100644 --- a/docs/job-metrics.md +++ b/docs/job-metrics.md @@ -2,6 +2,7 @@ | Metric name| Metric type | Labels/tags | Status | | ---------- | ----------- | ----------- | ----------- | +| kube_job_annotations | Gauge | `job_name`=<job-name>
`namespace`=<job-namespace>
`annotation_JOB_ANNOTATION`=<JOB_ANNOTATION> | EXPERIMENTAL | | kube_job_info | Gauge | `job_name`=<job-name>
`namespace`=<job-namespace> | STABLE | | kube_job_labels | Gauge | `job_name`=<job-name>
`namespace`=<job-namespace>
`label_JOB_LABEL`=<JOB_LABEL> | STABLE | | kube_job_owner | Gauge | `job_name`=<job-name>
`namespace`=<job-namespace>
`owner_kind`=<owner kind>
`owner_name`=<owner name>
`owner_is_controller`=<whether owner is controller> | STABLE | @@ -10,9 +11,9 @@ | kube_job_spec_active_deadline_seconds | Gauge | `job_name`=<job-name>
`namespace`=<job-namespace> | STABLE | | kube_job_status_active | Gauge | `job_name`=<job-name>
`namespace`=<job-namespace> | STABLE | | kube_job_status_succeeded | Gauge | `job_name`=<job-name>
`namespace`=<job-namespace> | STABLE | -| kube_job_status_failed | Gauge | `job_name`=<job-name>
`namespace`=<job-namespace> | STABLE | +| kube_job_status_failed | Gauge | `job_name`=<job-name>
`namespace`=<job-namespace>
`reason`=<failure reason> | STABLE | | kube_job_status_start_time | Gauge | `job_name`=<job-name>
`namespace`=<job-namespace> | STABLE | | kube_job_status_completion_time | Gauge | `job_name`=<job-name>
`namespace`=<job-namespace> | STABLE | -| kube_job_complete | Gauge | `job_name`=<job-name>
`namespace`=<job-namespace> | STABLE | -| kube_job_failed | Gauge | `job_name`=<job-name>
`namespace`=<job-namespace> | STABLE | +| kube_job_complete | Gauge | `job_name`=<job-name>
`namespace`=<job-namespace>
`condition`=<true\|false\|unknown> | STABLE | +| kube_job_failed | Gauge | `job_name`=<job-name>
`namespace`=<job-namespace>
`condition`=<true\|false\|unknown> | STABLE | | kube_job_created | Gauge | `job_name`=<job-name>
`namespace`=<job-namespace> | STABLE | diff --git a/docs/lease-metrics.md b/docs/lease-metrics.md new file mode 100644 index 0000000000..ee6bfb9ba7 --- /dev/null +++ b/docs/lease-metrics.md @@ -0,0 +1,6 @@ +# Lease Metrics + +| Metric name| Metric type | Labels/tags | Status | +| ---------- | ----------- |-------------------------------------------------------------------------------------------------------------------------------------------| ----------- | +| kube_lease_owner | Gauge | `lease`=<lease-name>
`owner_kind`=<onwer kind>
`owner_name`=<owner name>
`namespace` = <namespace>
`lease_holder`=<lease holder name>| EXPERIMENTAL | +| kube_lease_renew_time | Gauge | `lease`=<lease-name> | EXPERIMENTAL | diff --git a/docs/mutatingwebhookconfiguration.md b/docs/mutatingwebhookconfiguration-metrics.md similarity index 100% rename from docs/mutatingwebhookconfiguration.md rename to docs/mutatingwebhookconfiguration-metrics.md diff --git a/docs/namespace-metrics.md b/docs/namespace-metrics.md index 144ca2844f..5bea8fa8d3 100644 --- a/docs/namespace-metrics.md +++ b/docs/namespace-metrics.md @@ -2,7 +2,8 @@ | Metric name| Metric type | Labels/tags | Status | | ---------- | ----------- | ----------- | ----------- | +| kube_namespace_annotations | Gauge | `namespace`=<namespace-name>
`label_NS_ANNOTATION`=<NS_ANNOTATION> | EXPERIMENTAL | | kube_namespace_created | Gauge | `namespace`=<namespace-name> | STABLE | | kube_namespace_labels | Gauge | `namespace`=<namespace-name>
`label_NS_LABEL`=<NS_LABEL> | STABLE | | kube_namespace_status_condition | Gauge | `namespace`=<namespace-name>
`condition`=<NamespaceDeletionDiscoveryFailure\|NamespaceDeletionContentFailure\|NamespaceDeletionGroupVersionParsingFailure>
`status`=<true\|false\|unknown> | EXPERIMENTAL | -| kube_namespace_status_phase| Gauge | `namespace`=<namespace-name>
`status`=<Active\|Terminating> | STABLE | +| kube_namespace_status_phase| Gauge | `namespace`=<namespace-name>
`phase`=<Active\|Terminating> | STABLE | diff --git a/docs/networkpolicy-metrics.md b/docs/networkpolicy-metrics.md index 2a023d46c7..c5fa1f64f0 100644 --- a/docs/networkpolicy-metrics.md +++ b/docs/networkpolicy-metrics.md @@ -3,6 +3,7 @@ | Metric name | Metric type | Labels/tags | Status | | ------------------------------------- | ----------- | ------------------------------------------------------------------------------ | ------------ | +| kube_networkpolicy_annotations | Gauge | `namespace`=<namespace name> `networkpolicy`=<networkpolicy name> | EXPERIMENTAL | | kube_networkpolicy_created | Gauge | `namespace`=<namespace name> `networkpolicy`=<networkpolicy name> | EXPERIMENTAL | | kube_networkpolicy_labels | Gauge | `namespace`=<namespace name> `networkpolicy`=<networkpolicy name> | EXPERIMENTAL | | kube_networkpolicy_spec_egress_rules | Gauge | `namespace`=<namespace name> `networkpolicy`=<networkpolicy name> | EXPERIMENTAL | diff --git a/docs/node-metrics.md b/docs/node-metrics.md index 9effd04e4b..4004adaf17 100644 --- a/docs/node-metrics.md +++ b/docs/node-metrics.md @@ -1,20 +1,15 @@ # Node Metrics -| Metric name| Metric type | Labels/tags | Status | -| ---------- | ----------- | ----------- | ----------- | -| kube_node_info | Gauge | `node`=<node-address>
`kernel_version`=<kernel-version>
`os_image`=<os-image-name>
`container_runtime_version`=<container-runtime-and-version-combination>
`kubelet_version`=<kubelet-version>
`kubeproxy_version`=<kubeproxy-version>
`pod_cidr`=<pod-cidr>
`provider_id`=<provider-id> | STABLE | -| kube_node_labels | Gauge | `node`=<node-address>
`label_NODE_LABEL`=<NODE_LABEL> | STABLE | -| kube_node_role | Gauge | `node`=<node-address>
`role`=<NODE_ROLE> | EXPERIMENTAL | -| kube_node_spec_unschedulable | Gauge | `node`=<node-address>| -| kube_node_spec_taint | Gauge | `node`=<node-address>
`key`=<taint-key>
`value=`<taint-value>
`effect=`<taint-effect> | STABLE | -| kube_node_status_phase| Gauge | `node`=<node-address>
`phase`=<Pending\|Running\|Terminated> | DEPRECATED | -| kube_node_status_capacity | Gauge | `node`=<node-address>
`resource`=<resource-name>
`unit=`<resource-unit>| STABLE | -| kube_node_status_capacity_cpu_cores | Gauge | `node`=<node-address>| DEPRECATED | -| kube_node_status_capacity_memory_bytes | Gauge | `node`=<node-address>| DEPRECATED | -| kube_node_status_capacity_pods | Gauge | `node`=<node-address>| DEPRECATED | -| kube_node_status_allocatable | Gauge | `node`=<node-address>
`resource`=<resource-name>
`unit=`<resource-unit>| STABLE | -| kube_node_status_allocatable_cpu_cores | Gauge | `node`=<node-address>| DEPRECATED | -| kube_node_status_allocatable_memory_bytes | Gauge | `node`=<node-address>| DEPRECATED | -| kube_node_status_allocatable_pods | Gauge | `node`=<node-address>| DEPRECATED | -| kube_node_status_condition | Gauge | `node`=<node-address>
`condition`=<node-condition>
`status`=<true\|false\|unknown> | STABLE | -| kube_node_created | Gauge | `node`=<node-address>| STABLE | +| Metric name| Metric type | Description | Unit (where applicable) | Labels/tags | Status | +| ---------- | ----------- | ----------- | ----------------------- | ----------- | ------ | +| kube_node_annotations | Gauge | Kubernetes annotations converted to Prometheus labels | | `node`=<node-address>
`annotation_NODE_ANNOTATION`=<NODE_ANNOTATION> | EXPERIMENTAL | +| kube_node_info | Gauge | Information about a cluster node| |`node`=<node-address>
`kernel_version`=<kernel-version>
`os_image`=<os-image-name>
`container_runtime_version`=<container-runtime-and-version-combination>
`kubelet_version`=<kubelet-version>
`kubeproxy_version`=<kubeproxy-version>
`pod_cidr`=<pod-cidr>
`provider_id`=<provider-id>
`system_uuid`=<system-uuid>
`internal_ip`=<internal-ip> | STABLE | +| kube_node_labels | Gauge | Kubernetes labels converted to Prometheus labels | | `node`=<node-address>
`label_NODE_LABEL`=<NODE_LABEL> | STABLE | +| kube_node_role | Gauge | The role of a cluster node | | `node`=<node-address>
`role`=<NODE_ROLE> | EXPERIMENTAL | +| kube_node_spec_unschedulable | Gauge | Whether a node can schedule new pods | | `node`=<node-address>| STABLE | +| kube_node_spec_taint | Gauge | The taint of a cluster node. | |`node`=<node-address>
`key`=<taint-key>
`value=`<taint-value>
`effect=`<taint-effect> | STABLE | +| kube_node_status_capacity | Gauge | The total amount of resources available for a node | `cpu`=<core>
`ephemeral_storage`=<byte>
`pods`=<integer>
`attachable_volumes_*`=<byte>
`hugepages_*`=<byte>
`memory`=<byte> |`node`=<node-address>
`resource`=<resource-name>
`unit`=<resource-unit>| STABLE | +| kube_node_status_allocatable | Gauge | The amount of resources allocatable for pods (after reserving some for system daemons) | `cpu`=<core>
`ephemeral_storage`=<byte>
`pods`=<integer>
`attachable_volumes_*`=<byte>
`hugepages_*`=<byte>
`memory`=<byte> |`node`=<node-address>
`resource`=<resource-name>
`unit`=<resource-unit>| STABLE | +| kube_node_status_condition | Gauge | The condition of a cluster node | |`node`=<node-address>
`condition`=<node-condition>
`status`=<true\|false\|unknown> | STABLE | +| kube_node_created | Gauge | Unix creation timestamp | seconds |`node`=<node-address>| STABLE | +| kube_node_deletion_timestamp | Gauge | Unix deletion timestamp | seconds |`node`=<node-address>| EXPERIMENTAL | diff --git a/docs/persistentvolume-metrics.md b/docs/persistentvolume-metrics.md index 28a9e7aea1..a80e045768 100644 --- a/docs/persistentvolume-metrics.md +++ b/docs/persistentvolume-metrics.md @@ -1,9 +1,12 @@ # PersistentVolume Metrics -| Metric name| Metric type | Labels/tags | Status | -| ---------- | ----------- | ----------- | ----------- | -| kube_persistentvolume_capacity_bytes | Gauge | `persistentvolume`=<pv-name> | STABLE | -| kube_persistentvolume_status_phase | Gauge | `persistentvolume`=<pv-name>
`phase`=<Bound\|Failed\|Pending\|Available\|Released>| STABLE | -| kube_persistentvolume_labels | Gauge | `persistentvolume`=<persistentvolume-name>
`label_PERSISTENTVOLUME_LABEL`=<PERSISTENTVOLUME_LABEL> | STABLE | -| kube_persistentvolume_info | Gauge | `persistentvolume`=<pv-name>
`storageclass`=<storageclass-name> | STABLE | +| Metric name | Metric type | Description | Unit (where applicable) | Labels/tags | Status | +| ----------- | ----------- | ----------- | ----------- | ----------- | ------------ | +| kube_persistentvolume_annotations | Gauge | | | `persistentvolume`=<persistentvolume-name>
`annotation_PERSISTENTVOLUME_ANNOTATION`=<PERSISTENTVOLUME_ANNOTATION> | EXPERIMENTAL | +| kube_persistentvolume_capacity_bytes | Gauge | | | `persistentvolume`=<pv-name> | STABLE | +| kube_persistentvolume_status_phase | Gauge | | | `persistentvolume`=<pv-name>
`phase`=<Bound\|Failed\|Pending\|Available\|Released> | STABLE | +| kube_persistentvolume_claim_ref | Gauge | | | `persistentvolume`=<pv-name>
`claim_namespace`=<>
`name`=<> | STABLE | +| kube_persistentvolume_labels | Gauge | | | `persistentvolume`=<persistentvolume-name>
`label_PERSISTENTVOLUME_LABEL`=<PERSISTENTVOLUME_LABEL> | STABLE | +| kube_persistentvolume_info | Gauge | | | `persistentvolume`=<pv-name>
`storageclass`=<storageclass-name>
`gce_persistent_disk_name`=<pd-name>
`host_path`=<path-of-a-host-volume>
`host_path_type`=<host-mount-type>
`ebs_volume_id`=<ebs-volume-id>
`azure_disk_name`=<azure-disk-name>
`fc_wwids`=<fc-wwids-comma-separated>
`fc_lun`=<fc-lun>
`fc_target_wwns`=<fc-target-wwns-comma-separated>
`iscsi_target_portal`=<iscsi-target-portal>
`iscsi_iqn`=<iscsi-iqn>
`iscsi_lun`=<iscsi-lun>
`iscsi_initiator_name`=<iscsi-initiator-name>
`local_path`=<path-of-a-local-volume>
`local_fs`=<local-volume-fs-type>
`nfs_server`=<nfs-server>
`nfs_path`=<nfs-path>
`csi_driver`=<csi-driver>
`csi_volume_handle`=<csi-volume-handle> | STABLE | +| kube_persistentvolume_created | Gauge | Unix Creation Timestamp | seconds | `persistentvolume`=<persistentvolume-name>
| EXPERIMENTAL | diff --git a/docs/persistentvolumeclaim-metrics.md b/docs/persistentvolumeclaim-metrics.md index 40025b2ca2..2c92bc8247 100644 --- a/docs/persistentvolumeclaim-metrics.md +++ b/docs/persistentvolumeclaim-metrics.md @@ -1,14 +1,16 @@ # PersistentVolumeClaim Metrics -| Metric name| Metric type | Labels/tags | Status | -| ---------- | ----------- | ----------- | ----------- | -| kube_persistentvolumeclaim_access_mode | Gauge | `access_mode`=<persistentvolumeclaim-access-mode>
`namespace`=<persistentvolumeclaim-namespace>
`persistentvolumeclaim`=<persistentvolumeclaim-name> | STABLE | -| kube_persistentvolumeclaim_info | Gauge | `namespace`=<persistentvolumeclaim-namespace>
`persistentvolumeclaim`=<persistentvolumeclaim-name>
`storageclass`=<persistentvolumeclaim-storageclassname>
`volumename`=<volumename> | STABLE | -| kube_persistentvolumeclaim_labels | Gauge | `persistentvolumeclaim`=<persistentvolumeclaim-name>
`namespace`=<persistentvolumeclaim-namespace>
`label_PERSISTENTVOLUMECLAIM_LABEL`=<PERSISTENTVOLUMECLAIM_LABEL> | STABLE | -| kube_persistentvolumeclaim_resource_requests_storage_bytes | Gauge | `namespace`=<persistentvolumeclaim-namespace>
`persistentvolumeclaim`=<persistentvolumeclaim-name> | STABLE | -| kube_persistentvolumeclaim_status_condition | Gauge | `namespace` =<persistentvolumeclaim-namespace>
`persistentvolumeclaim`=<persistentvolumeclaim-name>
`type`=<persistentvolumeclaim-condition-type>
`status`=<true\|false\|unknown> | EXPERIMENTAL | -| kube_persistentvolumeclaim_status_phase | Gauge | `namespace`=<persistentvolumeclaim-namespace>
`persistentvolumeclaim`=<persistentvolumeclaim-name>
`phase`=<Pending\|Bound\|Lost> | STABLE | +| Metric name | Metric type | Description | Unit (where applicable) | Labels/tags | Status | +| ----------- | ------------- | ----------- | ----------- | ----------- | ----------- | +| kube_persistentvolumeclaim_annotations | Gauge | | | `persistentvolumeclaim`=<persistentvolumeclaim-name>
`namespace`=<persistentvolumeclaim-namespace>
`annotation_PERSISTENTVOLUMECLAIM_ANNOTATION`=<PERSISTENTVOLUMECLAIM_ANNOATION> | EXPERIMENTAL | +| kube_persistentvolumeclaim_access_mode | Gauge | | | `access_mode`=<persistentvolumeclaim-access-mode>
`namespace`=<persistentvolumeclaim-namespace>
`persistentvolumeclaim`=<persistentvolumeclaim-name> | STABLE | +| kube_persistentvolumeclaim_info | Gauge | | | `namespace`=<persistentvolumeclaim-namespace>
`persistentvolumeclaim`=<persistentvolumeclaim-name>
`storageclass`=<persistentvolumeclaim-storageclassname>
`volumename`=<volumename> | STABLE | +| kube_persistentvolumeclaim_labels | Gauge | | | `persistentvolumeclaim`=<persistentvolumeclaim-name>
`namespace`=<persistentvolumeclaim-namespace>
`label_PERSISTENTVOLUMECLAIM_LABEL`=<PERSISTENTVOLUMECLAIM_LABEL> | STABLE | +| kube_persistentvolumeclaim_resource_requests_storage_bytes | Gauge | | | `namespace`=<persistentvolumeclaim-namespace>
`persistentvolumeclaim`=<persistentvolumeclaim-name> | STABLE | +| kube_persistentvolumeclaim_status_condition | Gauge | | | `namespace` =<persistentvolumeclaim-namespace>
`persistentvolumeclaim`=<persistentvolumeclaim-name>
`type`=<persistentvolumeclaim-condition-type>
`status`=<true\false\unknown> | EXPERIMENTAL | +| kube_persistentvolumeclaim_status_phase | Gauge | | | `namespace`=<persistentvolumeclaim-namespace>
`persistentvolumeclaim`=<persistentvolumeclaim-name>
`phase`=<Pending\Bound\Lost> | STABLE | +| kube_persistentvolumeclaim_created | Gauge | Unix Creation Timestamp | seconds | `namespace`=<persistentvolumeclaim-namespace>
`persistentvolumeclaim`=<persistentvolumeclaim-name> | EXPERIMENTAL | Note: -- A special `` string will be used if PVC has no storage class. +- An empty string will be used if PVC has no storage class. diff --git a/docs/pod-metrics.md b/docs/pod-metrics.md index c315bca971..e6b8e96743 100644 --- a/docs/pod-metrics.md +++ b/docs/pod-metrics.md @@ -1,43 +1,83 @@ # Pod Metrics -| Metric name| Metric type | Labels/tags | Status | -| ---------- | ----------- | ----------- | ----------- | -| kube_pod_info | Gauge | `pod`=<pod-name>
`namespace`=<pod-namespace>
`host_ip`=<host-ip>
`pod_ip`=<pod-ip>
`node`=<node-name>
`created_by_kind`=<created_by_kind>
`created_by_name`=<created_by_name>
`uid`=<pod-uid>
`priority_class`=<priority_class>| STABLE | -| kube_pod_start_time | Gauge | `pod`=<pod-name>
`namespace`=<pod-namespace> | -| kube_pod_completion_time | Gauge | `pod`=<pod-name>
`namespace`=<pod-namespace> | STABLE | -| kube_pod_owner | Gauge | `pod`=<pod-name>
`namespace`=<pod-namespace>
`owner_kind`=<owner kind>
`owner_name`=<owner name>
`owner_is_controller`=<whether owner is controller> | STABLE | -| kube_pod_labels | Gauge | `pod`=<pod-name>
`namespace`=<pod-namespace>
`label_POD_LABEL`=<POD_LABEL> | STABLE | -| kube_pod_status_phase | Gauge | `pod`=<pod-name>
`namespace`=<pod-namespace>
`phase`=<Pending\|Running\|Succeeded\|Failed\|Unknown> | STABLE | -| kube_pod_status_ready | Gauge | `pod`=<pod-name>
`namespace`=<pod-namespace>
`condition`=<true\|false\|unknown> | STABLE | -| kube_pod_status_scheduled | Gauge | `pod`=<pod-name>
`namespace`=<pod-namespace>
`condition`=<true\|false\|unknown> | STABLE | -| kube_pod_container_info | Gauge | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`image`=<image-name>
`image_id`=<image-id>
`container_id`=<containerid> | STABLE | -| kube_pod_container_status_waiting | Gauge | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace> | STABLE | -| kube_pod_container_status_waiting_reason | Gauge | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<ContainerCreating\|CrashLoopBackOff\|ErrImagePull\|ImagePullBackOff\|CreateContainerConfigError\|InvalidImageName\|CreateContainerError> | STABLE | -| kube_pod_container_status_running | Gauge | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace> | STABLE | -| kube_pod_container_status_terminated | Gauge | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace> | STABLE | -| kube_pod_container_status_terminated_reason | Gauge | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<OOMKilled\|Error\|Completed\|ContainerCannotRun\|DeadlineExceeded> | STABLE | -| kube_pod_container_status_last_terminated_reason | Gauge | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<OOMKilled\|Error\|Completed\|ContainerCannotRun\|DeadlineExceeded> | STABLE | -| kube_pod_container_status_ready | Gauge | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace> | STABLE | -| kube_pod_container_status_restarts_total | Counter | `container`=<container-name>
`namespace`=<pod-namespace>
`pod`=<pod-name> | STABLE | -| kube_pod_container_resource_requests_cpu_cores | Gauge | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name> | DEPRECATED | -| kube_pod_container_resource_requests | Gauge | `resource`=<resource-name>
`unit`=<resource-unit>
`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name> | STABLE | -| kube_pod_container_resource_requests_memory_bytes | Gauge | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name> | DEPRECATED | -| kube_pod_container_resource_limits_cpu_cores | Gauge | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name> | DEPRECATED | -| kube_pod_container_resource_limits | Gauge | `resource`=<resource-name>
`unit`=<resource-unit>
`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name> | STABLE | -| kube_pod_container_resource_limits_memory_bytes | Gauge | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name> | DEPRECATED | -| kube_pod_created | Gauge | `pod`=<pod-name>
`namespace`=<pod-namespace> | -| kube_pod_restart_policy | Gauge | `pod`=<pod-name>
`namespace`=<pod-namespace>
`type`=<Always|Never|OnFailure> | STABLE | -| kube_pod_init_container_info | Gauge | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`image`=<image-name>
`image_id`=<image-id>
`container_id`=<containerid> | STABLE | -| kube_pod_init_container_status_waiting | Gauge | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace> | STABLE | -| kube_pod_init_container_status_waiting_reason | Gauge | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<ContainerCreating\|CrashLoopBackOff\|ErrImagePull\|ImagePullBackOff\|CreateContainerConfigError> | STABLE | -| kube_pod_init_container_status_running | Gauge | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace> | STABLE | -| kube_pod_init_container_status_terminated | Gauge | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace> | STABLE | -| kube_pod_init_container_status_terminated_reason | Gauge | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<OOMKilled\|Error\|Completed\|ContainerCannotRun\|DeadlineExceeded> | STABLE | -| kube_pod_init_container_status_last_terminated_reason | Gauge | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<OOMKilled\|Error\|Completed\|ContainerCannotRun\|DeadlineExceeded> | STABLE | -| kube_pod_init_container_status_ready | Gauge | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace> | STABLE | -| kube_pod_init_container_status_restarts_total | Counter | `container`=<container-name>
`namespace`=<pod-namespace>
`pod`=<pod-name> | STABLE | -| kube_pod_init_container_resource_limits | Gauge | `resource`=<resource-name>
`unit`=<resource-unit>
`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name> | STABLE | -| kube_pod_spec_volumes_persistentvolumeclaims_info | Gauge | `pod`=<pod-name>
`namespace`=<pod-namespace>
`volume`=<volume-name>
`persistentvolumeclaim`=<persistentvolumeclaim-claimname> | STABLE | -| kube_pod_spec_volumes_persistentvolumeclaims_readonly | Gauge | `pod`=<pod-name>
`namespace`=<pod-namespace>
`volume`=<volume-name>
`persistentvolumeclaim`=<persistentvolumeclaim-claimname> | STABLE | -| kube_pod_status_scheduled_time | Gauge | `pod`=<pod-name>
`namespace`=<pod-namespace> | STABLE | -| kube_pod_status_unschedulable | Gauge | `pod`=<pod-name>
`namespace`=<pod-namespace> | STABLE | +| Metric name| Metric type | Description | Unit (where applicable) | Labels/tags | Status | Opt-in | +| ---------- | ----------- |-----------------------------------------------------------------------| ----------------------- | ----------- | ------ | ------ | +| kube_pod_annotations | Gauge | Kubernetes annotations converted to Prometheus labels | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`annotation_POD_ANNOTATION`=<POD_ANNOTATION>
`uid`=<pod-uid> | EXPERIMENTAL | - +| kube_pod_info | Gauge | Information about pod | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`host_ip`=<host-ip>
`pod_ip`=<pod-ip>
`node`=<node-name>
`created_by_kind`=<created_by_kind>
`created_by_name`=<created_by_name>
`uid`=<pod-uid>
`priority_class`=<priority_class>
`host_network`=<host_network>| STABLE | - | +| kube_pod_ips | Gauge | Pod IP addresses | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`ip`=<pod-ip-address>
`ip_family`=<4 OR 6>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_start_time | Gauge | Start time in unix timestamp for a pod | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_completion_time | Gauge | Completion time in unix timestamp for a pod | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_owner | Gauge | Information about the Pod's owner | |`pod`=<pod-name>
`namespace`=<pod-namespace>
`owner_kind`=<owner kind>
`owner_name`=<owner name>
`owner_is_controller`=<whether owner is controller>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_labels | Gauge | Kubernetes labels converted to Prometheus labels | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`label_POD_LABEL`=<POD_LABEL>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_nodeselectors| Gauge | Describes the Pod nodeSelectors | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`nodeselector_NODE_SELECTOR`=<NODE_SELECTOR>
`uid`=<pod-uid> | EXPERIMENTAL | Opt-in | +| kube_pod_status_phase | Gauge | The pods current phase | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`phase`=<Pending\|Running\|Succeeded\|Failed\|Unknown>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_status_qos_class | Gauge | The pods current qosClass | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`qos_class`=<BestEffort\|Burstable\|Guaranteed>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_status_ready | Gauge | Describes whether the pod is ready to serve requests | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`condition`=<true\|false\|unknown>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_status_scheduled | Gauge | Describes the status of the scheduling process for the pod | |`pod`=<pod-name>
`namespace`=<pod-namespace>
`condition`=<true\|false\|unknown>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_info | Gauge | Information about a container in a pod | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`image`=<image-name>
`image_id`=<image-id>
`image_spec`=<image-spec>
`container_id`=<containerid>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_status_waiting | Gauge | Describes whether the container is currently in waiting state | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_status_waiting_reason | Gauge | Describes the reason the container is currently in waiting state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<container-waiting-reason>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_status_running | Gauge | Describes whether the container is currently in running state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_state_started | Gauge | Start time in unix timestamp for a pod container | seconds |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_status_terminated | Gauge | Describes whether the container is currently in terminated state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_status_terminated_reason | Gauge | Describes the reason the container is currently in terminated state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<container-terminated-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_container_status_last_terminated_reason | Gauge | Describes the last reason the container was in terminated state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<last-terminated-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_container_status_last_terminated_exitcode | Gauge | Describes the exit code for the last container in terminated state. | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_container_status_ready | Gauge | Describes whether the containers readiness check succeeded | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_status_ready_time | Gauge | Time when pod passed readiness probes. | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | +| kube_pod_status_container_ready_time | Gauge | Time when the container of the pod entered Ready state. | seconds | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | +| kube_pod_container_status_restarts_total | Counter | The number of container restarts per container | | `container`=<container-name>
`namespace`=<pod-namespace>
`pod`=<pod-name>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_container_resource_requests | Gauge | The number of requested request resource by a container. It is recommended to use the `kube_pod_resource_requests` metric exposed by kube-scheduler instead, as it is more precise. | `cpu`=<core>
`memory`=<bytes> |`resource`=<resource-name>
`unit`=<resource-unit>
`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_container_resource_limits | Gauge | The number of requested limit resource by a container. It is recommended to use the `kube_pod_resource_limits` metric exposed by kube-scheduler instead, as it is more precise. | `cpu`=<core>
`memory`=<bytes> |`resource`=<resource-name>
`unit`=<resource-unit>
`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_overhead_cpu_cores | Gauge | The pod overhead in regards to cpu cores associated with running a pod | core |`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_overhead_memory_bytes | Gauge | The pod overhead in regards to memory associated with running a pod | bytes |`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_runtimeclass_name_info | Gauge | The runtimeclass associated with the pod | |`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_created | Gauge | Unix creation timestamp | seconds |`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_deletion_timestamp | Gauge | Unix deletion timestamp | seconds |`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_restart_policy | Gauge | Describes the restart policy in use by this pod | |`pod`=<pod-name>
`namespace`=<pod-namespace>
`type`=<Always\|Never\|OnFailure>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_init_container_info | Gauge | Information about an init container in a pod | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`image`=<image-name>
`image_id`=<image-id>
`image_spec`=<image-spec>
`container_id`=<containerid>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_init_container_status_waiting | Gauge | Describes whether the init container is currently in waiting state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_init_container_status_waiting_reason | Gauge | Describes the reason the init container is currently in waiting state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<container-waiting-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_init_container_status_running | Gauge | Describes whether the init container is currently in running state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_init_container_status_terminated | Gauge | Describes whether the init container is currently in terminated state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_init_container_status_terminated_reason | Gauge | Describes the reason the init container is currently in terminated state | | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<container-terminated-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_init_container_status_last_terminated_reason | Gauge | Describes the last reason the init container was in terminated state | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<last-terminated-reason>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_init_container_status_ready | Gauge | Describes whether the init containers readiness check succeeded | |`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_init_container_status_restarts_total | Counter | The number of restarts for the init container | integer |`container`=<container-name>
`namespace`=<pod-namespace>
`pod`=<pod-name>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_init_container_resource_limits | Gauge | The number of CPU cores requested limit by an init container | `cpu`=<core>
`memory`=<bytes> |`resource`=<resource-name>
`unit`=<resource-unit>
`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_init_container_resource_requests | Gauge | The number of CPU cores requested by an init container | `cpu`=<core>
`memory`=<bytes> |`resource`=<resource-name>
`unit`=<resource-unit>
`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_spec_volumes_persistentvolumeclaims_info | Gauge | Information about persistentvolumeclaim volumes in a pod | |`pod`=<pod-name>
`namespace`=<pod-namespace>
`volume`=<volume-name>
`persistentvolumeclaim`=<persistentvolumeclaim-claimname>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_spec_volumes_persistentvolumeclaims_readonly | Gauge | Describes whether a persistentvolumeclaim is mounted read only | bool |`pod`=<pod-name>
`namespace`=<pod-namespace>
`volume`=<volume-name>
`persistentvolumeclaim`=<persistentvolumeclaim-claimname>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_status_reason | Gauge | The pod status reasons | |`pod`=<pod-name>
`namespace`=<pod-namespace>
`reason`=<Evicted\|NodeAffinity\|NodeLost\|Shutdown\|UnexpectedAdmissionError>
`uid`=<pod-uid> | EXPERIMENTAL | - | +| kube_pod_status_scheduled_time | Gauge | Unix timestamp when pod moved into scheduled status | seconds |`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_status_unschedulable | Gauge | Describes the unschedulable status for the pod | |`pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid> | STABLE | - | +| kube_pod_tolerations | Gauge | Information about the pod tolerations | | `pod`=<pod-name>
`namespace`=<pod-namespace>
`uid`=<pod-uid>
`key`=<toleration-key>
`operator`=<toleration-operator>
`value`=<toleration-value>
`effect`=<toleration-effect> `toleration_seconds`=<toleration-seconds> | EXPERIMENTAL | - | + +## Useful metrics queries + +### How to retrieve non-standard Pod state + +It is not straightforward to get the Pod states for certain cases like "Terminating" and "Unknown" since it is not stored behind a field in the `Pod.Status`. + +So to mimic the [logic](https://github.com/kubernetes/kubernetes/blob/v1.17.3/pkg/printers/internalversion/printers.go#L624) used by the `kubectl` command line, you will need to compose multiple metrics. + +For example: + +* To get the list of pods that are in the `Unknown` state, you can run the following PromQL query: `sum(kube_pod_status_phase{phase="Unknown"}) by (namespace, pod) or (count(kube_pod_deletion_timestamp) by (namespace, pod) * sum(kube_pod_status_reason{reason="NodeLost"}) by(namespace, pod))` + +* For Pods in `Terminating` state: `count(kube_pod_deletion_timestamp) by (namespace, pod) * count(kube_pod_status_reason{reason="NodeLost"} == 0) by (namespace, pod)` + +Here is an example of a Prometheus rule that can be used to alert on a Pod that has been in the `Terminated` state for more than `5m`. + +```yaml +groups: +- name: Pod state + rules: + - alert: PodsBlockInTerminatingState + expr: count(kube_pod_deletion_timestamp) by (namespace, pod) * count(kube_pod_status_reason{reason="NodeLost"} == 0) by (namespace, pod) > 0 + for: 5m + labels: + severity: page + annotations: + summary: Pod {{$labels.namespace}}/{{$labels.pod}} block in Terminating state. +``` diff --git a/docs/poddisruptionbudget-metrics.md b/docs/poddisruptionbudget-metrics.md index ae5668f968..0a18622e16 100644 --- a/docs/poddisruptionbudget-metrics.md +++ b/docs/poddisruptionbudget-metrics.md @@ -2,6 +2,8 @@ | Metric name| Metric type | Labels/tags | Status | | ---------- | ----------- | ----------- | ----------- | +| kube_poddisruptionbudget_annotations | Gauge | `poddisruptionbudget`=<poddisruptionbudget-name>
`namespace`=<poddisruptionbudget-namespace>
`annotation_PODDISRUPTIONBUDGET_ANNOTATION`=<PODDISRUPTIONBUDGET_ANNOATION> | EXPERIMENTAL | +| kube_poddisruptionbudget_labels | Gauge | `poddisruptionbudget`=<poddisruptionbudget-name>
`namespace`=<poddisruptionbudget-namespace>
`label_PODDISRUPTIONBUDGET_LABEL`=<PODDISRUPTIONBUDGET_ANNOATION> | EXPERIMENTAL | | kube_poddisruptionbudget_created | Gauge | `poddisruptionbudget`=<pdb-name>
`namespace`=<pdb-namespace> | STABLE | kube_poddisruptionbudget_status_current_healthy | Gauge | `poddisruptionbudget`=<pdb-name>
`namespace`=<pdb-namespace> | STABLE | kube_poddisruptionbudget_status_desired_healthy | Gauge | `poddisruptionbudget`=<pdb-name>
`namespace`=<pdb-namespace> | STABLE diff --git a/docs/replicaset-metrics.md b/docs/replicaset-metrics.md index 611eac87bd..d4bb9c38d0 100644 --- a/docs/replicaset-metrics.md +++ b/docs/replicaset-metrics.md @@ -2,12 +2,13 @@ | Metric name| Metric type | Labels/tags | Status | | ---------- | ----------- | ----------- | ----------- | +| kube_replicaset_annotations | Gauge | `replicaset`=<replicaset-name>
`namespace`=<replicaset-namespace>
`annotation_REPLICASET_ANNOTATION`=<REPLICASET_ANNOTATION> | EXPERIMENTAL | | kube_replicaset_status_replicas | Gauge | `replicaset`=<replicaset-name>
`namespace`=<replicaset-namespace> | STABLE | | kube_replicaset_status_fully_labeled_replicas | Gauge | `replicaset`=<replicaset-name>
`namespace`=<replicaset-namespace> | STABLE | | kube_replicaset_status_ready_replicas | Gauge | `replicaset`=<replicaset-name>
`namespace`=<replicaset-namespace> | STABLE | | kube_replicaset_status_observed_generation | Gauge | `replicaset`=<replicaset-name>
`namespace`=<replicaset-namespace> | STABLE | | kube_replicaset_spec_replicas | Gauge | `replicaset`=<replicaset-name>
`namespace`=<replicaset-namespace> | STABLE | | kube_replicaset_metadata_generation | Gauge | `replicaset`=<replicaset-name>
`namespace`=<replicaset-namespace> | STABLE | -| kube_replicaset_labels | Gauge | `replicaset`=<replicaset-name>
`namespace`=<replicaset-namespace> | STABLE | +| kube_replicaset_labels | Gauge | `replicaset`=<replicaset-name>
`namespace`=<replicaset-namespace>
`label_REPLICASET_LABEL`=<REPLICASET_LABEL> | STABLE | | kube_replicaset_created | Gauge | `replicaset`=<replicaset-name>
`namespace`=<replicaset-namespace> | STABLE | | kube_replicaset_owner | Gauge | `replicaset`=<replicaset-name>
`namespace`=<replicaset-namespace>
`owner_kind`=<owner kind>
`owner_name`=<owner name>
`owner_is_controller`=<whether owner is controller> | STABLE | diff --git a/docs/replicationcontroller-metrics.md b/docs/replicationcontroller-metrics.md index 2000245c4d..e8882c1e5f 100644 --- a/docs/replicationcontroller-metrics.md +++ b/docs/replicationcontroller-metrics.md @@ -10,3 +10,4 @@ | kube_replicationcontroller_spec_replicas | Gauge | `replicationcontroller`=<replicationcontroller-name>
`namespace`=<replicationcontroller-namespace> | STABLE | | kube_replicationcontroller_metadata_generation | Gauge | `replicationcontroller`=<replicationcontroller-name>
`namespace`=<replicationcontroller-namespace> | STABLE | | kube_replicationcontroller_created | Gauge | `replicationcontroller`=<replicationcontroller-name>
`namespace`=<replicationcontroller-namespace> | STABLE | +| kube_replicationcontroller_owner | Gauge | `replicationcontroller`=<replicationcontroller-name>
`namespace`=<replicationcontroller-namespace>
`owner_kind`=<owner kind>
`owner_name`=<owner name>
`owner_is_controller`=<whether owner is controller> | EXPERIMENTAL | diff --git a/docs/role-metrics.md b/docs/role-metrics.md new file mode 100644 index 0000000000..fc52087c10 --- /dev/null +++ b/docs/role-metrics.md @@ -0,0 +1,9 @@ +# Role Metrics + +| Metric name| Metric type | Labels/tags | Status | +| ---------- | ----------- | ----------- | ----------- | +| kube_role_annotations | Gauge | `role`=<role-name>
`namespace`=<role-namespace> | EXPERIMENTAL +| kube_role_labels | Gauge | `role`=<role-name>
`namespace`=<role-namespace> | EXPERIMENTAL +| kube_role_info | Gauge | `role`=<role-name>
`namespace`=<role-namespace> | EXPERIMENTAL +| kube_role_created | Gauge | `role`=<role-name>
`namespace`=<role-namespace> | EXPERIMENTAL | +| kube_role_metadata_resource_version | Gauge | `role`=<role-name>
`namespace`=<role-namespace> | EXPERIMENTAL | diff --git a/docs/rolebinding-metrics.md b/docs/rolebinding-metrics.md new file mode 100644 index 0000000000..2a0b6986d0 --- /dev/null +++ b/docs/rolebinding-metrics.md @@ -0,0 +1,9 @@ +# RoleBinding Metrics + +| Metric name| Metric type | Labels/tags | Status | +| ---------- | ----------- | ----------- | ----------- | +| kube_rolebinding_annotations | Gauge | `rolebinding`=<rolebinding-name>
`namespace`=<rolebinding-namespace> | EXPERIMENTAL +| kube_rolebinding_labels | Gauge | `rolebinding`=<rolebinding-name>
`namespace`=<rolebinding-namespace> | EXPERIMENTAL +| kube_rolebinding_info | Gauge | `rolebinding`=<rolebinding-name>
`namespace`=<rolebinding-namespace>
`roleref_kind`=<role-kind>
`roleref_name`=<role-name>| EXPERIMENTAL +| kube_rolebinding_created | Gauge | `rolebinding`=<rolebinding-name>
`namespace`=<rolebinding-namespace> | EXPERIMENTAL | +| kube_rolebinding_metadata_resource_version | Gauge | `rolebinding`=<rolebinding-name>
`namespace`=<rolebinding-namespace> | EXPERIMENTAL | diff --git a/docs/secret-metrics.md b/docs/secret-metrics.md index 780df876de..b5c4beeed6 100644 --- a/docs/secret-metrics.md +++ b/docs/secret-metrics.md @@ -2,6 +2,7 @@ | Metric name| Metric type | Labels/tags | Status | | ---------- | ----------- | ----------- | ----------- | +| kube_secret_annotations | Gauge | `secret`=<secret-name>
`namespace`=<secret-namespace>
`annotations_SECRET_ANNOTATION`=<SECRET_ANNOTATION> | EXPERIMENTAL | | kube_secret_info | Gauge | `secret`=<secret-name>
`namespace`=<secret-namespace> | STABLE | | kube_secret_type | Gauge | `secret`=<secret-name>
`namespace`=<secret-namespace>
`type`=<secret-type> | STABLE | | kube_secret_labels | Gauge | `secret`=<secret-name>
`namespace`=<secret-namespace>
`label_SECRET_LABEL`=<SECRET_LABEL> | STABLE | diff --git a/docs/service-metrics.md b/docs/service-metrics.md index 822144113f..e4619869c6 100644 --- a/docs/service-metrics.md +++ b/docs/service-metrics.md @@ -1,10 +1,11 @@ # Service Metrics -| Metric name| Metric type | Labels/tags | Status | -| ---------- | ----------- | ----------- | ----------- | -| kube_service_info | Gauge | `service`=<service-name>
`namespace`=<service-namespace>
`cluster_ip`=<service cluster ip>
`external_name`=<service external name> `load_balancer_ip`=<service load balancer ip> | STABLE | -| kube_service_labels | Gauge | `service`=<service-name>
`namespace`=<service-namespace>
`label_SERVICE_LABEL`=<SERVICE_LABEL> | STABLE | -| kube_service_created | Gauge | `service`=<service-name>
`namespace`=<service-namespace> | STABLE | -| kube_service_spec_type | Gauge | `service`=<service-name>
`namespace`=<service-namespace>
`type`=<ClusterIP\|NodePort\|LoadBalancer\|ExternalName> | STABLE | -| kube_service_spec_external_ip | Gauge | `service`=<service-name>
`namespace`=<service-namespace>
`external_ip`=<external-ip> | STABLE | -| kube_service_status_load_balancer_ingress | Gauge | `service`=<service-name>
`namespace`=<service-namespace>
`ip`=<load-balancer-ingress-ip>
`hostname`=<load-balancer-ingress-hostname> | STABLE | +| Metric name| Metric type | Description | Unit (where applicable) | Labels/tags | Status | +| ---------- | ----------- | ----------- | ----------------------- | ----------- | ------ | +| kube_service_annotations | Gauge | Kubernetes annotations converted to Prometheus labels | |`service`=<service-name>
`namespace`=<service-namespace>
`uid`=<service-uid>
`annotation_SERVICE_ANNOTATION`=<SERVICE_ANNOTATION> | EXPERIMENTAL | +| kube_service_info | Gauge | Information about service | |`service`=<service-name>
`namespace`=<service-namespace>
`uid`=<service-uid>
`cluster_ip`=<service cluster ip>
`external_name`=<service external name>
`load_balancer_ip`=<service load balancer ip> | STABLE | +| kube_service_labels | Gauge | Kubernetes labels converted to Prometheus labels | |`service`=<service-name>
`namespace`=<service-namespace>
`uid`=<service-uid>
`label_SERVICE_LABEL`=<SERVICE_LABEL> | STABLE | +| kube_service_created | Gauge | Unix creation timestamp | seconds |`service`=<service-name>
`namespace`=<service-namespace>
`uid`=<service-uid> | STABLE | +| kube_service_spec_type | Gauge | Type about service | |`service`=<service-name>
`namespace`=<service-namespace>
`uid`=<service-uid>
`type`=<ClusterIP\|NodePort\|LoadBalancer\|ExternalName> | STABLE | +| kube_service_spec_external_ip | Gauge | Service external ips. One series for each ip | |`service`=<service-name>
`namespace`=<service-namespace>
`uid`=<service-uid>
`external_ip`=<external-ip> | STABLE | +| kube_service_status_load_balancer_ingress | Gauge | Service load balancer ingress status | |`service`=<service-name>
`namespace`=<service-namespace>
`uid`=<service-uid>
`ip`=<load-balancer-ingress-ip>
`hostname`=<load-balancer-ingress-hostname> | STABLE | diff --git a/docs/serviceaccount-metrics.md b/docs/serviceaccount-metrics.md new file mode 100644 index 0000000000..2dd4881170 --- /dev/null +++ b/docs/serviceaccount-metrics.md @@ -0,0 +1,11 @@ +# ServiceAccount Metrics + +| Metric name | Metric type | Description | Unit (where applicable) | Labels/tags | Status | +|---------------------------------------|-------------|--------------------------------------------------------------------------------|-------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------| +| kube_serviceaccount_info | Gauge | Information about a service account | | `namespace`=<serviceaccount-namespace>
`serviceaccount`=<serviceaccount-name>
`uid`=<serviceaccount-uid>
`automount_token`=<serviceaccount-automount-token> | EXPERIMENTAL | +| kube_serviceaccount_created | Gauge | Unix creation timestamp | | `namespace`=<serviceaccount-namespace>
`serviceaccount`=<serviceaccount-name>
`uid`=<serviceaccount-uid> | EXPERIMENTAL | +| kube_serviceaccount_deleted | Gauge | Unix deletion timestamp | | `namespace`=<serviceaccount-namespace>
`serviceaccount`=<serviceaccount-name>
`uid`=<serviceaccount-uid> | EXPERIMENTAL | +| kube_serviceaccount_secret | Gauge | Secret being referenced by a service account | | `namespace`=<serviceaccount-namespace>
`serviceaccount`=<serviceaccount-name>
`uid`=<serviceaccount-uid>
`name`=<secret-name> | EXPERIMENTAL | +| kube_serviceaccount_image_pull_secret | Gauge | Secret being referenced by a service account for the purpose of pulling images | | `namespace`=<serviceaccount-namespace>
`serviceaccount`=<serviceaccount-name>
`uid`=<serviceaccount-uid>
`name`=<secret-name> | EXPERIMENTAL | +| kube_serviceaccount_annotations | Gauge | Kubernetes annotations converted to Prometheus labels | | `namespace`=<serviceaccount-namespace>
`serviceaccount`=<serviceaccount-name>
`uid`=<serviceaccount-uid>
`annotation_SERVICE_ACCOUNT_ANNOTATION`=<SERVICE_ACCOUNT_ANNOTATION> | EXPERIMENTAL | +| kube_serviceaccount_labels | Gauge | Kubernetes labels converted to Prometheus labels | | `namespace`=<serviceaccount-namespace>
`serviceaccount`=<serviceaccount-name>
`uid`=<serviceaccount-uid>
`label_SERVICE_ACCOUNT_LABEL`=<SERVICE_ACCOUNT_LABEL> | EXPERIMENTAL | diff --git a/docs/statefulset-metrics.md b/docs/statefulset-metrics.md index 175bafe729..e7d0aa567e 100644 --- a/docs/statefulset-metrics.md +++ b/docs/statefulset-metrics.md @@ -2,14 +2,17 @@ | Metric name| Metric type | Labels/tags | Status | | ---------- | ----------- | ----------- | ----------- | +| kube_statefulset_annotations | Gauge | `statefulset`=<statefulset-name>
`namespace`=<statefulset-namespace>
`annotation_STATEFULSET_ANNOTATION`=<STATEFULSET_ANNOTATION> | EXPERIMENTAL | | kube_statefulset_status_replicas | Gauge | `statefulset`=<statefulset-name>
`namespace`=<statefulset-namespace> | STABLE | | kube_statefulset_status_replicas_current | Gauge | `statefulset`=<statefulset-name>
`namespace`=<statefulset-namespace> | STABLE | | kube_statefulset_status_replicas_ready | Gauge | `statefulset`=<statefulset-name>
`namespace`=<statefulset-namespace> | STABLE | +| kube_statefulset_status_replicas_available | Gauge | `statefulset`=<statefulset-name>
`namespace`=<statefulset-namespace> | EXPERIMENTAL | | kube_statefulset_status_replicas_updated | Gauge | `statefulset`=<statefulset-name>
`namespace`=<statefulset-namespace> | STABLE | | kube_statefulset_status_observed_generation | Gauge | `statefulset`=<statefulset-name>
`namespace`=<statefulset-namespace> | STABLE | | kube_statefulset_replicas | Gauge | `statefulset`=<statefulset-name>
`namespace`=<statefulset-namespace> | STABLE | | kube_statefulset_metadata_generation | Gauge | `statefulset`=<statefulset-name>
`namespace`=<statefulset-namespace> | STABLE | +| kube_statefulset_persistentvolumeclaim_retention_policy | Gauge | `statefulset`=<statefulset-name>
`namespace`=<statefulset-namespace>
`when_deleted`=<statefulset-when-deleted-pvc-policy>
`when_scaled`=<statefulset-when-scaled-pvc-policy> | EXPERIMENTAL | | kube_statefulset_created | Gauge | `statefulset`=<statefulset-name>
`namespace`=<statefulset-namespace> | STABLE | | kube_statefulset_labels | Gauge | `statefulset`=<statefulset-name>
`namespace`=<statefulset-namespace>
`label_STATEFULSET_LABEL`=<STATEFULSET_LABEL> | STABLE | | kube_statefulset_status_current_revision | Gauge | `statefulset`=<statefulset-name>
`namespace`=<statefulset-namespace>
`revision`=<statefulset-current-revision> | STABLE | -| kube_statefulset_status_update_revision | Gauge | `statefulset`=<statefulset-name>
`namespace`=<statefulset-namespace>
`revision`=<statefulset-update-revision> | STABLE | +| kube_statefulset_status_update_revision | Gauge | `statefulset`=<statefulset-name>
`namespace`=<statefulset-namespace>
`revision`=<statefulset-update-revision> | STABLE | diff --git a/docs/storageclass-metrics.md b/docs/storageclass-metrics.md index 71ed49ea85..d5f4088fb8 100644 --- a/docs/storageclass-metrics.md +++ b/docs/storageclass-metrics.md @@ -2,6 +2,7 @@ | Metric name| Metric type | Labels/tags | Status | | ---------- | ----------- | ----------- | ----------- | -| kube_storageclass_info | Gauge | `storageclass`=<storageclass-name>
`provisioner`=<storageclass-provisioner>
`reclaimPolicy`=<storageclass-reclaimPolicy>
`volumeBindingMode`=<storageclass-volumeBindingMode> | STABLE | +| kube_storageclass_annotations | Gauge | `storageclass`=<storageclass-name>
`annotation_STORAGECLASS_ANNOTATION`=<STORAGECLASS_ANNOTATION> | EXPERIMENTAL | +| kube_storageclass_info | Gauge | `storageclass`=<storageclass-name>
`provisioner`=<storageclass-provisioner>
`reclaim_policy`=<storageclass-reclaimPolicy>
`volume_binding_mode`=<storageclass-volumeBindingMode> | STABLE | | kube_storageclass_labels | Gauge | `storageclass`=<storageclass-name>
`label_STORAGECLASS_LABEL`=<STORAGECLASS_LABEL> | STABLE | | kube_storageclass_created | Gauge | `storageclass`=<storageclass-name> | STABLE | diff --git a/docs/validatingwebhookconfiguration.md b/docs/validatingwebhookconfiguration-metrics.md similarity index 100% rename from docs/validatingwebhookconfiguration.md rename to docs/validatingwebhookconfiguration-metrics.md diff --git a/docs/verticalpodautoscaler-metrics.md b/docs/verticalpodautoscaler-metrics.md index 1ee49bf97b..e715af7e88 100644 --- a/docs/verticalpodautoscaler-metrics.md +++ b/docs/verticalpodautoscaler-metrics.md @@ -1,12 +1,145 @@ # Vertical Pod Autoscaler Metrics -| Metric name | Metric type | Labels/tags | Status | -| -------------------------------- | ----------- | ------------------------------------------------------------- | ------ | -| kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed | Gauge | `container`=<container name>
`namespace`=<namespace>
`resource`=<cpu | memory>
`target_api_version`=<api version>
`target_kind`=<target kind>
`target_name`=<target name>
`unit`=<core | byte>
`verticalpodautoscaler`=<vertical pod autoscaler name> | EXPERIMENTAL | -| kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed | Gauge | `container`=<container name>
`namespace`=<namespace>
`resource`=<cpu | memory>
`target_api_version`=<api version>
`target_kind`=<target kind>
`target_name`=<target name>
`unit`=<core | byte>
`verticalpodautoscaler`=<vertical pod autoscaler name> | EXPERIMENTAL | -| kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound | Gauge | `container`=<container name>
`namespace`=<namespace>
`resource`=<cpu | memory>
`target_api_version`=<api version>
`target_kind`=<target kind>
`target_name`=<target name>
`unit`=<core | byte>
`verticalpodautoscaler`=<vertical pod autoscaler name> | EXPERIMENTAL | -| kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target | Gauge | `container`=<container name>
`namespace`=<namespace>
`resource`=<cpu | memory>
`target_api_version`=<api version>
`target_kind`=<target kind>
`target_name`=<target name>
`unit`=<core | byte>
`verticalpodautoscaler`=<vertical pod autoscaler name> | EXPERIMENTAL | -| kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget | Gauge | `container`=<container name>
`namespace`=<namespace>
`resource`=<cpu | memory>
`target_api_version`=<api version>
`target_kind`=<target kind>
`target_name`=<target name>
`unit`=<core | byte>
`verticalpodautoscaler`=<vertical pod autoscaler name> | EXPERIMENTAL | -| kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound | Gauge | `container`=<container name>
`namespace`=<namespace>
`resource`=<cpu | memory>
`target_api_version`=<api version>
`target_kind`=<target kind>
`target_name`=<target name>
`unit`=<core | byte>
`verticalpodautoscaler`=<vertical pod autoscaler name> | EXPERIMENTAL | -| kube_verticalpodautoscaler_labels | Gauge | `label_app`=<foo>
`namespace`=<namespace>
`target_api_version`=<api version>
`target_kind`=<target kind>
`target_name`=<target name>
`verticalpodautoscaler`=<vertical pod autoscaler name> | EXPERIMENTAL | -| kube_verticalpodautoscaler_spec_updatepolicy_updatemode | Gauge | `namespace`=<namespace>
`target_api_version`=<api version>
`target_kind`=<target kind>
`target_name`=<target name>
`update_mode`=<foo>
`verticalpodautoscaler`=<vertical pod autoscaler name> | EXPERIMENTAL | +## DEPRECATION NOTICE + +From v2.9.0 onwards, `vericalpodautoscalers` will be removed from the list of default resources. This means that specifying that in the `--resource` flag will **not** generate metrics for the same. In order to generate `verticalpodautoscalers` metrics, you will have to explicitly specify it in `--custom-resource-state-config*` (either the inline yaml, or the configuration file), like so: +```yaml +# Using --resource=verticalpodautoscalers, we get the following output: +# HELP kube_verticalpodautoscaler_annotations (Deprecated since v2.9.0) Kubernetes annotations converted to Prometheus labels. +# TYPE kube_verticalpodautoscaler_annotations gauge +# kube_verticalpodautoscaler_annotations{namespace="default",verticalpodautoscaler="hamster-vpa",target_api_version="apps/v1",target_kind="Deployment",target_name="hamster"} 1 +# A similar result can be achieved by specifying the following in --custom-resource-state-config: +kind: CustomResourceStateMetrics +spec: + resources: + - groupVersionKind: + group: autoscaling.k8s.io + kind: "VerticalPodAutoscaler" + version: "v1" + labelsFromPath: + verticalpodautoscaler: [metadata, name] + namespace: [metadata, namespace] + target_api_version: [apiVersion] + target_kind: [spec, targetRef, kind] + target_name: [spec, targetRef, name] + metrics: + - name: "annotations" + help: "Kubernetes annotations converted to Prometheus labels." + each: + type: Gauge + gauge: + path: [metadata, annotations] +# This will output the following metric: +# HELP kube_customresource_autoscaling_annotations Kubernetes annotations converted to Prometheus labels. +# TYPE kube_customresource_autoscaling_annotations gauge +# kube_customresource_autoscaling_annotations{customresource_group="autoscaling.k8s.io", customresource_kind="VerticalPodAutoscaler", customresource_version="v1", namespace="default",target_api_version="autoscaling.k8s.io/v1",target_kind="Deployment",target_name="hamster",verticalpodautoscaler="hamster-vpa"} 123 +``` +PS. The above configuration was tested on [this](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/examples/hamster.yaml) VPA configuration, with an added annotation (`foo: 123`). +*** + +| Metric name | Metric type | Labels/tags | Status | +| -------------------------------- | ----------- | ------------------------------------------------------------- | ------ | +| kube_verticalpodautoscaler_annotations | Gauge | `annotation_app`=<foo>
`namespace`=<namespace>
`target_api_version`=<api version>
`target_kind`=<target kind>
`target_name`=<target name>
`verticalpodautoscaler`=<vertical pod autoscaler name> | DEPRECATED | +| kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed | Gauge | `container`=<container name>
`namespace`=<namespace>
`resource`=<cpu memory>
`target_api_version`=<api version>
`target_kind`=<target kind>
`target_name`=<target name>
`unit`=<core byte>
`verticalpodautoscaler`=<vertical pod autoscaler name> | DEPRECATED | +| kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed | Gauge | `container`=<container name>
`namespace`=<namespace>
`resource`=<cpu memory>
`target_api_version`=<api version>
`target_kind`=<target kind>
`target_name`=<target name>
`unit`=<core byte>
`verticalpodautoscaler`=<vertical pod autoscaler name> | DEPRECATED | +| kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound | Gauge | `container`=<container name>
`namespace`=<namespace>
`resource`=<cpu memory>
`target_api_version`=<api version>
`target_kind`=<target kind>
`target_name`=<target name>
`unit`=<core byte>
`verticalpodautoscaler`=<vertical pod autoscaler name> | DEPRECATED | +| kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target | Gauge | `container`=<container name>
`namespace`=<namespace>
`resource`=<cpu memory>
`target_api_version`=<api version>
`target_kind`=<target kind>
`target_name`=<target name>
`unit`=<core byte>
`verticalpodautoscaler`=<vertical pod autoscaler name> | DEPRECATED | +| kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget | Gauge | `container`=<container name>
`namespace`=<namespace>
`resource`=<cpu memory>
`target_api_version`=<api version>
`target_kind`=<target kind>
`target_name`=<target name>
`unit`=<core byte>
`verticalpodautoscaler`=<vertical pod autoscaler name> | DEPRECATED | +| kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound | Gauge | `container`=<container name>
`namespace`=<namespace>
`resource`=<cpu memory>
`target_api_version`=<api version>
`target_kind`=<target kind>
`target_name`=<target name>
`unit`=<core byte>
`verticalpodautoscaler`=<vertical pod autoscaler name> | DEPRECATED | +| kube_verticalpodautoscaler_labels | Gauge | `label_app`=<foo>
`namespace`=<namespace>
`target_api_version`=<api version>
`target_kind`=<target kind>
`target_name`=<target name>
`verticalpodautoscaler`=<vertical pod autoscaler name> | DEPRECATED | +| kube_verticalpodautoscaler_spec_updatepolicy_updatemode | Gauge | `namespace`=<namespace>
`target_api_version`=<api version>
`target_kind`=<target kind>
`target_name`=<target name>
`update_mode`=<foo>
`verticalpodautoscaler`=<vertical pod autoscaler name> | DEPRECATED | + +## Configuration + +Vertical Pod Autoscalers(VPAs) are managed as custom resources. + +To enable the Vertical Pod Autoscaler collector, please: + +1. Ensure that the Vertical Pod Autoscaler CRDs are installed in the cluster. The CRDs are [here](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/deploy/vpa-beta2-crd.yaml). +2. Ensure that `verticalpodautoscalers` is included in list of `Resources` enabled using the flag `--resources` when `kube-state-metrics` is run (see below). + +One of the [command line arguments](./docs/cli-arguments.md) for `kube-state-metrics` is `--resources`. If this flag is omitted, a default set of Resources is enabled. This default list does **not** include Vertical Pod Autoscalers. + +To enable Vertical Pod Autoscalers, the `kube-state-metrics` flag `--resource` must be included when the binary is run and the list of resources must include `verticalpodautoscalers`. + + +### Examples + +The following configures `kube-state-metrics` on the command line and in the `args` section of a Kubernetes manifest. Because neither command includes the `--resource` flag, the default set of resources will be include **but** metrics for Vertical Pod Autoscalers will **not** be included: + +Shell: + +```bash +kube-state-metrics \ +--telemetry-port=8081 \ +--kubeconfig=... \ +--apiserver=... +``` + +Kubernetes: + +```YAML +spec: + template: + spec: + containers: + - args: + - --telemetry-port=8081 + - --kubeconfig=... + - --apiserver=... +``` + +To include Vertical Pod Autoscaler metrics, you must include the `--resources` flag and to include the default resources, you must include the list of default resources **and** `verticalpodautoscalers`, i.e.: + +Shell: + +```bash +kube-state-metrics \ +--telemetry-port=8081 \ +--kubeconfig=... \ +--apiserver=... \ +--resources=certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,horizontalpodautoscalers, ingresses,jobs,leases,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,verticalpodautoscalers,volumeattachments +``` + +Kubernetes: + +```YAML +spec: + template: + spec: + containers: + - args: + - --telemetry-port=8081 + - --kubeconfig=... + - --apiserver=... + - --resources=certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,horizontalpodautoscalers, ingresses,jobs,leases,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,verticalpodautoscalers,volumeattachments +``` + +### Confirmation + +To confirm that a `kube-state-metrics` process includes `verticalpodautoscalers`, you can: + +Shell: + +```bash +ps aux \ +| grep kube-state-metrics \ +| grep verticalpodautoscalers +``` + +Kubernetes: assuming your deployment is called `kube-state-metrics`: + +```bash +DEPLOYMENT="kube-state-metrics" +NAMESPACE="default" + +kubectl get deployment/${DEPLOYMENT} \ +--namespace=${NAMESPACE} \ +--output=jsonpath="{range .spec.template.spec.containers[?(@.name=='kube-state-metrics')].args[*]}{@}{'\n'}{end}" +``` + +Should include (among other `--flags`): + +```console +--resources=certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,horizontalpodautoscalers,ingresses,jobs,leases,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,verticalpodautoscalers,volumeattachments +``` diff --git a/docs/volumeattachment-metrics.md b/docs/volumeattachment-metrics.md index db5905a058..bae3102e03 100644 --- a/docs/volumeattachment-metrics.md +++ b/docs/volumeattachment-metrics.md @@ -1,8 +1,8 @@ -# PersistentVolume Metrics +# VolumeAttachment Metrics | Metric name| Metric type | Labels/tags | Status | | ---------- | ----------- | ----------- | ----------- | -| kube_volumeattachment_info | Gauge | `volumeattachment`=<volumeattachment-name>
`attacher`=<attacher-name>
`nodeName`=<node-name> | EXPERIMENTAL | +| kube_volumeattachment_info | Gauge | `volumeattachment`=<volumeattachment-name>
`attacher`=<attacher-name>
`node`=<node-name> | EXPERIMENTAL | | kube_volumeattachment_created | Gauge | `volumeattachment`=<volumeattachment-name> | EXPERIMENTAL | | kube_volumeattachment_labels | Gauge | `volumeattachment`=<volumeattachment-name>
`label_VOLUMEATTACHMENT_LABEL`=<VOLUMEATTACHMENT_LABEL> | EXPERIMENTAL | | kube_volumeattachment_spec_source_persistentvolume | Gauge | `volumeattachment`=<volumeattachment-name>
`volumename`=<persistentvolume-name> | EXPERIMENTAL | diff --git a/examples/autosharding/cluster-role-binding.yaml b/examples/autosharding/cluster-role-binding.yaml index 6e0ee41932..d331654ea1 100644 --- a/examples/autosharding/cluster-role-binding.yaml +++ b/examples/autosharding/cluster-role-binding.yaml @@ -2,8 +2,9 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: + app.kubernetes.io/component: exporter app.kubernetes.io/name: kube-state-metrics - app.kubernetes.io/version: v1.9.7 + app.kubernetes.io/version: 2.8.2 name: kube-state-metrics roleRef: apiGroup: rbac.authorization.k8s.io diff --git a/examples/autosharding/cluster-role.yaml b/examples/autosharding/cluster-role.yaml index c04db29000..d18d3742a3 100644 --- a/examples/autosharding/cluster-role.yaml +++ b/examples/autosharding/cluster-role.yaml @@ -2,8 +2,9 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: + app.kubernetes.io/component: exporter app.kubernetes.io/name: kube-state-metrics - app.kubernetes.io/version: v1.9.7 + app.kubernetes.io/version: 2.8.2 name: kube-state-metrics rules: - apiGroups: @@ -14,6 +15,7 @@ rules: - nodes - pods - services + - serviceaccounts - resourcequotas - replicationcontrollers - limitranges @@ -24,16 +26,6 @@ rules: verbs: - list - watch -- apiGroups: - - extensions - resources: - - daemonsets - - deployments - - replicasets - - ingresses - verbs: - - list - - watch - apiGroups: - apps resources: @@ -85,6 +77,13 @@ rules: verbs: - list - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch - apiGroups: - storage.k8s.io resources: @@ -105,6 +104,25 @@ rules: - networking.k8s.io resources: - networkpolicies + - ingressclasses + - ingresses + verbs: + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + - clusterroles + - rolebindings + - roles verbs: - list - watch diff --git a/examples/autosharding/kustomization.yaml b/examples/autosharding/kustomization.yaml new file mode 100644 index 0000000000..2cffbff155 --- /dev/null +++ b/examples/autosharding/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: kube-system + +resources: + - cluster-role-binding.yaml + - cluster-role.yaml + - role-binding.yaml + - role.yaml + - service-account.yaml + - service.yaml + - statefulset.yaml diff --git a/examples/autosharding/role-binding.yaml b/examples/autosharding/role-binding.yaml index 0aa0dde3c0..953ac33d8c 100644 --- a/examples/autosharding/role-binding.yaml +++ b/examples/autosharding/role-binding.yaml @@ -2,9 +2,11 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: + app.kubernetes.io/component: exporter app.kubernetes.io/name: kube-state-metrics - app.kubernetes.io/version: v1.9.7 + app.kubernetes.io/version: 2.8.2 name: kube-state-metrics + namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role diff --git a/examples/autosharding/role.yaml b/examples/autosharding/role.yaml index 68986c4549..53829ad630 100644 --- a/examples/autosharding/role.yaml +++ b/examples/autosharding/role.yaml @@ -2,8 +2,9 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: labels: + app.kubernetes.io/component: exporter app.kubernetes.io/name: kube-state-metrics - app.kubernetes.io/version: v1.9.7 + app.kubernetes.io/version: 2.8.2 name: kube-state-metrics namespace: kube-system rules: diff --git a/examples/autosharding/service-account.yaml b/examples/autosharding/service-account.yaml index e0f9abf25b..ef4919f908 100644 --- a/examples/autosharding/service-account.yaml +++ b/examples/autosharding/service-account.yaml @@ -1,8 +1,10 @@ apiVersion: v1 +automountServiceAccountToken: false kind: ServiceAccount metadata: labels: + app.kubernetes.io/component: exporter app.kubernetes.io/name: kube-state-metrics - app.kubernetes.io/version: v1.9.7 + app.kubernetes.io/version: 2.8.2 name: kube-state-metrics namespace: kube-system diff --git a/examples/autosharding/service.yaml b/examples/autosharding/service.yaml index f5b488f5ad..e2848a14cb 100644 --- a/examples/autosharding/service.yaml +++ b/examples/autosharding/service.yaml @@ -2,8 +2,9 @@ apiVersion: v1 kind: Service metadata: labels: + app.kubernetes.io/component: exporter app.kubernetes.io/name: kube-state-metrics - app.kubernetes.io/version: v1.9.7 + app.kubernetes.io/version: 2.8.2 name: kube-state-metrics namespace: kube-system spec: diff --git a/examples/autosharding/statefulset.yaml b/examples/autosharding/statefulset.yaml index c43f991e20..29cf1b3196 100644 --- a/examples/autosharding/statefulset.yaml +++ b/examples/autosharding/statefulset.yaml @@ -2,8 +2,9 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: + app.kubernetes.io/component: exporter app.kubernetes.io/name: kube-state-metrics - app.kubernetes.io/version: v1.9.7 + app.kubernetes.io/version: 2.8.2 name: kube-state-metrics namespace: kube-system spec: @@ -15,25 +16,25 @@ spec: template: metadata: labels: + app.kubernetes.io/component: exporter app.kubernetes.io/name: kube-state-metrics - app.kubernetes.io/version: v1.9.7 + app.kubernetes.io/version: 2.8.2 spec: + automountServiceAccountToken: true containers: - args: - --pod=$(POD_NAME) - --pod-namespace=$(POD_NAMESPACE) env: - name: POD_NAME - value: "" valueFrom: fieldRef: fieldPath: metadata.name - name: POD_NAMESPACE - value: "" valueFrom: fieldRef: fieldPath: metadata.namespace - image: quay.io/coreos/kube-state-metrics:v1.9.7 + image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.8.2 livenessProbe: httpGet: path: /healthz @@ -52,7 +53,13 @@ spec: port: 8081 initialDelaySeconds: 5 timeoutSeconds: 5 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 65534 nodeSelector: kubernetes.io/os: linux serviceAccountName: kube-state-metrics - volumeClaimTemplates: [] diff --git a/examples/prometheus-alerting-rules/alerts.yaml b/examples/prometheus-alerting-rules/alerts.yaml index f4b046e86d..ba80354da0 100644 --- a/examples/prometheus-alerting-rules/alerts.yaml +++ b/examples/prometheus-alerting-rules/alerts.yaml @@ -3,9 +3,8 @@ groups: rules: - alert: KubeStateMetricsListErrors annotations: - message: kube-state-metrics is experiencing errors at an elevated rate in list - operations. This is likely causing it to not be able to expose metrics about - Kubernetes objects correctly or at all. + description: kube-state-metrics is experiencing errors at an elevated rate in list operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + summary: kube-state-metrics is experiencing errors in list operations. expr: | (sum(rate(kube_state_metrics_list_total{job="kube-state-metrics",result="error"}[5m])) / @@ -16,9 +15,8 @@ groups: severity: critical - alert: KubeStateMetricsWatchErrors annotations: - message: kube-state-metrics is experiencing errors at an elevated rate in watch - operations. This is likely causing it to not be able to expose metrics about - Kubernetes objects correctly or at all. + description: kube-state-metrics is experiencing errors at an elevated rate in watch operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + summary: kube-state-metrics is experiencing errors in watch operations. expr: | (sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics",result="error"}[5m])) / @@ -27,3 +25,24 @@ groups: for: 15m labels: severity: critical + - alert: KubeStateMetricsShardingMismatch + annotations: + description: kube-state-metrics pods are running with different --total-shards configuration, some Kubernetes objects may be exposed multiple times or not exposed at all. + summary: kube-state-metrics sharding is misconfigured. + expr: | + stdvar (kube_state_metrics_total_shards{job="kube-state-metrics"}) != 0 + for: 15m + labels: + severity: critical + - alert: KubeStateMetricsShardsMissing + annotations: + description: kube-state-metrics shards are missing, some Kubernetes objects are not being exposed. + summary: kube-state-metrics shards are missing. + expr: | + 2^max(kube_state_metrics_total_shards{job="kube-state-metrics"}) - 1 + - + sum( 2 ^ max by (shard_ordinal) (kube_state_metrics_shard_ordinal{job="kube-state-metrics"}) ) + != 0 + for: 15m + labels: + severity: critical diff --git a/examples/standard/cluster-role-binding.yaml b/examples/standard/cluster-role-binding.yaml index 6e0ee41932..d331654ea1 100644 --- a/examples/standard/cluster-role-binding.yaml +++ b/examples/standard/cluster-role-binding.yaml @@ -2,8 +2,9 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: + app.kubernetes.io/component: exporter app.kubernetes.io/name: kube-state-metrics - app.kubernetes.io/version: v1.9.7 + app.kubernetes.io/version: 2.8.2 name: kube-state-metrics roleRef: apiGroup: rbac.authorization.k8s.io diff --git a/examples/standard/cluster-role.yaml b/examples/standard/cluster-role.yaml index c04db29000..d18d3742a3 100644 --- a/examples/standard/cluster-role.yaml +++ b/examples/standard/cluster-role.yaml @@ -2,8 +2,9 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: + app.kubernetes.io/component: exporter app.kubernetes.io/name: kube-state-metrics - app.kubernetes.io/version: v1.9.7 + app.kubernetes.io/version: 2.8.2 name: kube-state-metrics rules: - apiGroups: @@ -14,6 +15,7 @@ rules: - nodes - pods - services + - serviceaccounts - resourcequotas - replicationcontrollers - limitranges @@ -24,16 +26,6 @@ rules: verbs: - list - watch -- apiGroups: - - extensions - resources: - - daemonsets - - deployments - - replicasets - - ingresses - verbs: - - list - - watch - apiGroups: - apps resources: @@ -85,6 +77,13 @@ rules: verbs: - list - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch - apiGroups: - storage.k8s.io resources: @@ -105,6 +104,25 @@ rules: - networking.k8s.io resources: - networkpolicies + - ingressclasses + - ingresses + verbs: + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + - clusterroles + - rolebindings + - roles verbs: - list - watch diff --git a/examples/standard/deployment.yaml b/examples/standard/deployment.yaml index b6fa1fba96..c34e9c811d 100644 --- a/examples/standard/deployment.yaml +++ b/examples/standard/deployment.yaml @@ -2,8 +2,9 @@ apiVersion: apps/v1 kind: Deployment metadata: labels: + app.kubernetes.io/component: exporter app.kubernetes.io/name: kube-state-metrics - app.kubernetes.io/version: v1.9.7 + app.kubernetes.io/version: 2.8.2 name: kube-state-metrics namespace: kube-system spec: @@ -14,11 +15,13 @@ spec: template: metadata: labels: + app.kubernetes.io/component: exporter app.kubernetes.io/name: kube-state-metrics - app.kubernetes.io/version: v1.9.7 + app.kubernetes.io/version: 2.8.2 spec: + automountServiceAccountToken: true containers: - - image: quay.io/coreos/kube-state-metrics:v1.9.7 + - image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.8.2 livenessProbe: httpGet: path: /healthz @@ -37,6 +40,13 @@ spec: port: 8081 initialDelaySeconds: 5 timeoutSeconds: 5 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 65534 nodeSelector: kubernetes.io/os: linux serviceAccountName: kube-state-metrics diff --git a/examples/standard/service-account.yaml b/examples/standard/service-account.yaml index e0f9abf25b..ef4919f908 100644 --- a/examples/standard/service-account.yaml +++ b/examples/standard/service-account.yaml @@ -1,8 +1,10 @@ apiVersion: v1 +automountServiceAccountToken: false kind: ServiceAccount metadata: labels: + app.kubernetes.io/component: exporter app.kubernetes.io/name: kube-state-metrics - app.kubernetes.io/version: v1.9.7 + app.kubernetes.io/version: 2.8.2 name: kube-state-metrics namespace: kube-system diff --git a/examples/standard/service.yaml b/examples/standard/service.yaml index f5b488f5ad..e2848a14cb 100644 --- a/examples/standard/service.yaml +++ b/examples/standard/service.yaml @@ -2,8 +2,9 @@ apiVersion: v1 kind: Service metadata: labels: + app.kubernetes.io/component: exporter app.kubernetes.io/name: kube-state-metrics - app.kubernetes.io/version: v1.9.7 + app.kubernetes.io/version: 2.8.2 name: kube-state-metrics namespace: kube-system spec: diff --git a/go.mod b/go.mod index bc93be7428..77dbd01337 100644 --- a/go.mod +++ b/go.mod @@ -1,79 +1,89 @@ -module k8s.io/kube-state-metrics +module k8s.io/kube-state-metrics/v2 require ( - github.com/brancz/gojsontoyaml v0.0.0-20190425155809-e8bd32d46b3d - github.com/campoy/embedmd v1.0.0 - github.com/dgryski/go-jump v0.0.0-20170409065014-e1f439676b57 - github.com/google/go-jsonnet v0.14.0 - github.com/jsonnet-bundler/jsonnet-bundler v0.1.1-0.20190930114713-10e24cb86976 - github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.5.1 - github.com/prometheus/prometheus v2.5.0+incompatible - github.com/robfig/cron/v3 v3.0.0 - github.com/spf13/pflag v1.0.5 - golang.org/x/tools v0.0.0-20210106214847-113979e3529a - k8s.io/api v0.17.5 - k8s.io/apimachinery v0.17.5 - k8s.io/autoscaler/vertical-pod-autoscaler v0.0.0-20191115143342-4cf961056038 - k8s.io/client-go v0.17.5 - k8s.io/klog v1.0.0 + github.com/dgryski/go-jump v0.0.0-20211018200510-ba001c3ffce0 + github.com/fsnotify/fsnotify v1.6.0 + github.com/gobuffalo/flect v1.0.0 + github.com/google/go-cmp v0.5.9 + github.com/oklog/run v1.1.0 + github.com/prometheus/client_golang v1.14.0 + github.com/prometheus/client_model v0.3.0 + github.com/prometheus/common v0.39.0 + github.com/prometheus/exporter-toolkit v0.8.2 + github.com/robfig/cron/v3 v3.0.1 + github.com/spf13/cobra v1.6.1 + github.com/spf13/viper v1.15.0 + github.com/stretchr/testify v1.8.1 + gopkg.in/yaml.v3 v3.0.1 + k8s.io/api v0.26.1 + k8s.io/apimachinery v0.26.1 + k8s.io/autoscaler/vertical-pod-autoscaler v0.13.0 + k8s.io/client-go v0.26.1 + k8s.io/component-base v0.26.1 + k8s.io/klog/v2 v2.90.0 + k8s.io/sample-controller v0.26.1 + k8s.io/utils v0.0.0-20230202215443-34013725500c ) require ( - cloud.google.com/go v0.56.0 // indirect - github.com/Azure/go-autorest/autorest v0.10.0 // indirect - github.com/Azure/go-autorest/autorest/adal v0.8.3 // indirect - github.com/Azure/go-autorest/autorest/date v0.2.0 // indirect - github.com/Azure/go-autorest/logger v0.1.0 // indirect - github.com/Azure/go-autorest/tracing v0.5.0 // indirect - github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect - github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/coreos/go-systemd/v22 v22.4.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect - github.com/evanphx/json-patch v4.2.0+incompatible // indirect - github.com/fatih/color v1.9.0 // indirect - github.com/ghodss/yaml v1.0.0 // indirect - github.com/go-logr/logr v0.1.0 // indirect + github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/go-kit/log v0.2.1 // indirect + github.com/go-logfmt/logfmt v0.5.1 // indirect + github.com/go-logr/logr v1.2.3 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.20.0 // indirect + github.com/go-openapi/swag v0.19.14 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.4.0 // indirect - github.com/google/go-cmp v0.4.0 // indirect - github.com/google/gofuzz v1.0.0 // indirect - github.com/googleapis/gnostic v0.4.0 // indirect - github.com/gophercloud/gophercloud v0.10.0 // indirect - github.com/hashicorp/golang-lru v0.5.4 // indirect - github.com/imdario/mergo v0.3.5 // indirect - github.com/json-iterator/go v1.1.9 // indirect - github.com/mattn/go-colorable v0.1.6 // indirect - github.com/mattn/go-isatty v0.0.12 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/jpillora/backoff v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mailru/easyjson v0.7.6 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect + github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.9.1 // indirect - github.com/prometheus/procfs v0.0.11 // indirect - golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect - golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect - golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect - golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f // indirect - golang.org/x/text v0.3.3 // indirect - golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect - google.golang.org/appengine v1.6.6 // indirect - google.golang.org/protobuf v1.21.0 // indirect - gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect + github.com/prometheus/procfs v0.8.0 // indirect + github.com/rogpeppe/go-internal v1.8.1 // indirect + github.com/spf13/afero v1.9.3 // indirect + github.com/spf13/cast v1.5.0 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.4.2 // indirect + golang.org/x/crypto v0.0.0-20221012134737-56aed061732a // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/oauth2 v0.3.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/term v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect + golang.org/x/time v0.1.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.2.8 // indirect - k8s.io/klog/v2 v2.0.0 // indirect - k8s.io/kube-openapi v0.0.0-20200316234421-82d701f24f9d // indirect - k8s.io/utils v0.0.0-20200414100711-2df71ebbae66 // indirect - sigs.k8s.io/yaml v1.1.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect + sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect ) -replace ( - github.com/dgrijalva/jwt-go => github.com/dgrijalva/jwt-go v0.0.0-20210802184156-9742bd7fca1c - github.com/prometheus/prometheus => github.com/prometheus/prometheus v0.0.0-20200609090129-a6600f564e3c -) - -go 1.18 +go 1.20 diff --git a/go.sum b/go.sum index b0989e14c7..f2f1821ee1 100644 --- a/go.sum +++ b/go.sum @@ -1,717 +1,295 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.56.0 h1:WRz29PgAsVEyPSDHyk+0fpEkwEFyfhHn+JbksT6gIL4= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go v41.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest v0.10.0 h1:mvdtztBqcL8se7MdrUweNieTNi4kfNG6GOJuurQJpuY= -github.com/Azure/go-autorest/autorest v0.10.0/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= -github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= -github.com/Azure/go-autorest/autorest/adal v0.8.3 h1:O1AGG9Xig71FxdX9HO5pGNyZ7TbSyHaVg+5eJO/jSGw= -github.com/Azure/go-autorest/autorest/adal v0.8.3/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= -github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM= -github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= -github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc= -github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= -github.com/Azure/go-autorest/autorest/to v0.3.0/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= -github.com/Azure/go-autorest/autorest/validation v0.2.0/go.mod h1:3EEqHnBxQGHXRYq3HT1WyXAvT7LLY3tl70hw6tQIbjI= -github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= -github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= -github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= -github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= -github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= -github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= -github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= -github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.3.3/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= -github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= -github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= -github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.30.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= -github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= -github.com/brancz/gojsontoyaml v0.0.0-20190425155809-e8bd32d46b3d h1:DMb8SuAL9+demT8equqMMzD8C/uxqWmj4cgV7ufrpQo= -github.com/brancz/gojsontoyaml v0.0.0-20190425155809-e8bd32d46b3d/go.mod h1:IyUJYN1gvWjtLF5ZuygmxbnsAyP3aJS6cHzIuZY50B0= -github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= -github.com/campoy/embedmd v1.0.0 h1:V4kI2qTJJLf4J29RzI/MAt2c3Bl4dQSYPuflzwFH2hY= -github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8= -github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/cenkalti/backoff v0.0.0-20181003080854-62661b46c409/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= -github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= -github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= -github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/coreos/go-systemd/v22 v22.4.0 h1:y9YHcjnjynCd/DVbg5j9L/33jQM3MxJlbj/zWskzfGU= +github.com/coreos/go-systemd/v22 v22.4.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v0.0.0-20210802184156-9742bd7fca1c h1:OJBBa7basGFBjGwuNSRH8Iddfow24XcQ0DaQcM3YOXM= -github.com/dgrijalva/jwt-go v0.0.0-20210802184156-9742bd7fca1c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= -github.com/dgryski/go-jump v0.0.0-20170409065014-e1f439676b57 h1:qZNIK8jjHgLFHAW2wzCWPEv0ZIgcBhU7X3oDt/p3Sv0= -github.com/dgryski/go-jump v0.0.0-20170409065014-e1f439676b57/go.mod h1:4hKCXuwrJoYvHZxJ86+bRVTOMyJ0Ej+RqfSm8mHi6KA= -github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/dgryski/go-jump v0.0.0-20211018200510-ba001c3ffce0 h1:0wH6nO9QEa02Qx8sIQGw6ieKdz+BXjpccSOo9vXNl4U= +github.com/dgryski/go-jump v0.0.0-20211018200510-ba001c3ffce0/go.mod h1:4hKCXuwrJoYvHZxJ86+bRVTOMyJ0Ej+RqfSm8mHi6KA= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= -github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= +github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I= -github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= -github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= -github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= -github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= -github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= -github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ= -github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= -github.com/go-openapi/errors v0.19.3/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= -github.com/go-openapi/errors v0.19.4/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= -github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= -github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= -github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= -github.com/go-openapi/loads v0.19.3/go.mod h1:YVfqhUCdahYwR3f3iiwQLhicVRvLlU/WO5WPaZvcvSI= -github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= -github.com/go-openapi/loads v0.19.5/go.mod h1:dswLCAdonkRufe/gSUC3gN8nTSaB9uaS2es0x5/IbjY= -github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= -github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= -github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= -github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo= -github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= -github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/spec v0.19.6/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= -github.com/go-openapi/spec v0.19.7/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= -github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= -github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= -github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= -github.com/go-openapi/strfmt v0.19.4/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= -github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= -github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= -github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= +github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= -github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= -github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= -github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= -github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo= -github.com/go-openapi/validate v0.19.8/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= -github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= -github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= -github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= -github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= -github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= -github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= -github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= -github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= -github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= -github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= -github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= -github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= -github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= -github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= -github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= -github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/gobuffalo/flect v1.0.0 h1:eBFmskjXZgAOagiTXJH25Nt5sdFwNRcb8DKZsIsAUQI= +github.com/gobuffalo/flect v1.0.0/go.mod h1:l9V6xSb4BlXwsxEMj3FVEub2nkdQjWhPvD8XTTlHPQc= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= -github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-jsonnet v0.14.0 h1:as/sAfmjOHqY/OMBR4mv9I8ZY0/jNuqN3u44AicwxPs= -github.com/google/go-jsonnet v0.14.0/go.mod h1:zPGC9lj/TbjkBtUACIvYR/ILHrFqKRhxeEA+bLyeMnY= -github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= -github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200417002340-c6e0a841f49a/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.4.0 h1:BXDUo8p/DaxC+4FJY/SSx3gvnx9C1VdHNgaUkiEL5mk= -github.com/googleapis/gnostic v0.4.0/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= -github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= -github.com/gophercloud/gophercloud v0.10.0 h1:Et+UGxoD72pK6K+46uOwyVxbtXJ6KBkWAegXBmqlf6c= -github.com/gophercloud/gophercloud v0.10.0/go.mod h1:gmC5oQqMDOMO1t1gq5DquX/yAU808e/4mzjjDA76+Ss= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.14.4/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0= -github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= -github.com/hashicorp/consul/api v1.4.0/go.mod h1:xc8u05kyMa3Wjr9eEAsIAo3dg8+LywT5E/Cl7cNS5nU= -github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/consul/sdk v0.4.0/go.mod h1:fY08Y9z5SvJqevyZNy6WWPXiG3KwBPAvlcdx16zZ0fM= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v0.12.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.2.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/memberlist v0.2.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hashicorp/serf v0.9.0/go.mod h1:YL0HO+FifKOW2u1ke99DGVu1zhcpZzNwrLIqBC7vbYU= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/flux v0.65.0/go.mod h1:BwN2XG2lMszOoquQaFdPET8FRQfrXiZsWmcMO9rkaVY= -github.com/influxdata/influxdb v1.8.0/go.mod h1:SIzcnsjaHRFpmlxpJ4S3NT64qtEKYweNTUMb/vh0OMQ= -github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= -github.com/influxdata/influxql v1.1.0/go.mod h1:KpVI7okXjK6PRi3Z5B+mtKZli+R1DnZgb3N+tzevNgo= -github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE= -github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8= -github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= -github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= -github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jsonnet-bundler/jsonnet-bundler v0.1.1-0.20190930114713-10e24cb86976 h1:Yolu/ufxugb61pYkJUCTDNwvOYlMVRl/NBcxh3Nulw0= -github.com/jsonnet-bundler/jsonnet-bundler v0.1.1-0.20190930114713-10e24cb86976/go.mod h1:/by7P/OoohkI3q4CgSFqcoFsVY+IaNbzOVDknEsKDeU= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= -github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= -github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= -github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= -github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= -github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= -github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.6/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= -github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= -github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w= -github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= -github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= -github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= -github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= -github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= -github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= +github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= +github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= -github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= -github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/prometheus/alertmanager v0.20.0/go.mod h1:9g2i48FAyZW6BtbsnvHtMHQXl2aVtrORKwKVCQ+nbrg= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= -github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= -github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= -github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= -github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI= -github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/prometheus v0.0.0-20200609090129-a6600f564e3c h1:x3RZiripR6DQd/8nYVNzdOla4HfBHBKcfE9cO6w7dSI= -github.com/prometheus/prometheus v0.0.0-20200609090129-a6600f564e3c/go.mod h1:S5n0C6tSgdnwWshBUceRx5G1OsjLv/EeZ9t3wIfEtsY= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= -github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= -github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E= -github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI= +github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y= +github.com/prometheus/exporter-toolkit v0.8.2 h1:sbJAfBXQFkG6sUkbwBun8MNdzW9+wd5YfPYofbmj0YM= +github.com/prometheus/exporter-toolkit v0.8.2/go.mod h1:00shzmJL7KxcsabLWcONwpyNEuWhREOnFqZW7vadFS0= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/satori/go.uuid v0.0.0-20160603004225-b111a074d5ef/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= -github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= -github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= +github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= +github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= +github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= +github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= -github.com/uber/jaeger-client-go v2.23.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= -github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= -github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= -go.mongodb.org/mongo-driver v1.3.2/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= -go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200422194213-44a606286825/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20221012134737-56aed061732a h1:NmSIgad6KjE6VvHciPZuNRTKxGhlPfD6OA87W/PLkqg= +golang.org/x/crypto v0.0.0-20221012134737-56aed061732a/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= @@ -720,7 +298,6 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -733,6 +310,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -741,15 +319,10 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -759,31 +332,41 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8= +golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -792,80 +375,68 @@ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190310054646-10058d7d4faa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI= -golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= +golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -877,31 +448,19 @@ golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190813034749-528a2984e271/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191203134012-c197fd4bf371/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -909,25 +468,30 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200422205258-72e4a01eba43/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= -gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= -gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= -gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= -google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -937,146 +501,140 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200420144010-e5e8543f8aeb/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.0/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/fsnotify/fsnotify.v1 v1.4.7/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1qJ27s+7ltE= -gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= -gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.1.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.0.0-20191016110408-35e52d86657a/go.mod h1:/L5qH+AD540e7Cetbui1tuJeXdmNhO8jM6VkXeDdDhQ= -k8s.io/api v0.0.0-20191109101512-6d4d1612ba53/go.mod h1:VJq7+38rpM4TSUbRiZX4P5UVAKK2UQpNQLZClkFQkpE= -k8s.io/api v0.0.0-20191112020540-7f9008e52f64/go.mod h1:8svLRMiLwQReMTycutfjsaQ0ackWIf8HCT4UcixYLjI= -k8s.io/api v0.17.5 h1:EkVieIbn1sC8YCDwckLKLpf+LoVofXYW72+LTZWo4aQ= -k8s.io/api v0.17.5/go.mod h1:0zV5/ungglgy2Rlm3QK8fbxkXVs+BSJWpJP/+8gUVLY= -k8s.io/apimachinery v0.0.0-20191004115801-a2eda9f80ab8/go.mod h1:llRdnznGEAqC3DcNm6yEj472xaFVfLM7hnYofMb12tQ= -k8s.io/apimachinery v0.0.0-20191109100837-dffb012825f2/go.mod h1:+6CX7hP4aLfX2sb91JYDMIp0VqDSog2kZu0BHe+lP+s= -k8s.io/apimachinery v0.0.0-20191111054156-6eb29fdf75dc/go.mod h1:+6CX7hP4aLfX2sb91JYDMIp0VqDSog2kZu0BHe+lP+s= -k8s.io/apimachinery v0.17.5 h1:QAjfgeTtSGksdkgyaPrIb4lhU16FWMIzxKejYD5S0gc= -k8s.io/apimachinery v0.17.5/go.mod h1:ioIo1G/a+uONV7Tv+ZmCbMG1/a3kVw5YcDdncd8ugQ0= -k8s.io/autoscaler/vertical-pod-autoscaler v0.0.0-20191115143342-4cf961056038 h1:2AUVR4J8OIUM1UM1PshlfoXvXD4/XRZat9c/LIfFLOM= -k8s.io/autoscaler/vertical-pod-autoscaler v0.0.0-20191115143342-4cf961056038/go.mod h1:bo2qh32Y1lvDnTWVSlYibXIVVtwZMb0fYAqeyEWNEuI= -k8s.io/client-go v0.0.0-20191016111102-bec269661e48/go.mod h1:hrwktSwYGI4JK+TJA3dMaFyyvHVi/aLarVHpbs8bgCU= -k8s.io/client-go v0.0.0-20191109102209-3c0d1af94be5/go.mod h1:T9KDMwZhkD0ygfa5hs6lPRymsuj92WN84SowSq6gOEw= -k8s.io/client-go v0.17.5 h1:Sm/9AQ415xPAX42JLKbJZnreXFgD2rVfDUDwOTm0gzA= -k8s.io/client-go v0.17.5/go.mod h1:S8uZpBpjJJdEH/fEyxcqg7Rn0P5jH+ilkgBHjriSmNo= -k8s.io/code-generator v0.0.0-20191109100332-a9a0d9c0b3aa/go.mod h1:fRFrKVixH946mn5PeglV2fvxbE86JesGi16bsWZ1xz4= -k8s.io/component-base v0.0.0-20191016111319-039242c015a9/go.mod h1:SuWowIgd/dtU/m/iv8OD9eOxp3QZBBhTIiWMsBQvKjI= -k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.4.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/klog/v2 v2.0.0 h1:Foj74zO6RbjjP4hBEKjnYtjjAhGg4jNynUdYF6fJrok= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/kube-openapi v0.0.0-20200316234421-82d701f24f9d h1:jocF7XFucw2pEiv2wS7wk2FRFCjDFGV1oa4TMs0SAT0= -k8s.io/kube-openapi v0.0.0-20200316234421-82d701f24f9d/go.mod h1:F+5wygcW0wmRTnM3cOgIqGivxkwSWIWT5YdsDbeAOaU= -k8s.io/metrics v0.0.0-20191109111301-80b462294217/go.mod h1:GSVMuBbi34fc7MSnJ1Q//5ijIPF5ykXeEVQ49V5l2SM= -k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20200414100711-2df71ebbae66 h1:Ly1Oxdu5p5ZFmiVT71LFgeZETvMfZ1iBIGeOenT2JeM= -k8s.io/utils v0.0.0-20200414100711-2df71ebbae66/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= -modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= -modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= -modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= -modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.26.1 h1:f+SWYiPd/GsiWwVRz+NbFyCgvv75Pk9NK6dlkZgpCRQ= +k8s.io/api v0.26.1/go.mod h1:xd/GBNgR0f707+ATNyPmQ1oyKSgndzXij81FzWGsejg= +k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ= +k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= +k8s.io/autoscaler/vertical-pod-autoscaler v0.13.0 h1:pH6AsxeBZcyX6KBqcnl7SPIJqbN1d59RrEBuIE6Rq6c= +k8s.io/autoscaler/vertical-pod-autoscaler v0.13.0/go.mod h1:LraL5kR2xX7jb4VMCG6/tUH4I75uRHlnzC0VWQHcyWk= +k8s.io/client-go v0.26.1 h1:87CXzYJnAMGaa/IDDfRdhTzxk/wzGZ+/HUQpqgVSZXU= +k8s.io/client-go v0.26.1/go.mod h1:IWNSglg+rQ3OcvDkhY6+QLeasV4OYHDjdqeWkDQZwGE= +k8s.io/component-base v0.26.1 h1:4ahudpeQXHZL5kko+iDHqLj/FSGAEUnSVO0EBbgDd+4= +k8s.io/component-base v0.26.1/go.mod h1:VHrLR0b58oC035w6YQiBSbtsf0ThuSwXP+p5dD/kAWU= +k8s.io/klog/v2 v2.90.0 h1:VkTxIV/FjRXn1fgNNcKGM8cfmL1Z33ZjXRTVxKCoF5M= +k8s.io/klog/v2 v2.90.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= +k8s.io/sample-controller v0.26.1 h1:+t3EbW66zbQSFA378/Bfje6C68ZqjFy1sYPdLgZfkYE= +k8s.io/sample-controller v0.26.1/go.mod h1:f3gQsdfg38iReAcxh9IaHXVIdO+bEo8LKOzlX63rCP4= +k8s.io/utils v0.0.0-20230202215443-34013725500c h1:YVqDar2X7YiQa/DVAXFMDIfGF8uGrHQemlrwRU5NlVI= +k8s.io/utils v0.0.0-20230202215443-34013725500c/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= -sigs.k8s.io/structured-merge-diff/v2 v2.0.1/go.mod h1:Wb7vfKAodbKgf6tn1Kl0VvGj7mRH6DGaRcixXEJXTsE= -sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/internal/store/builder.go b/internal/store/builder.go index 22383c122c..7a3473a92f 100644 --- a/internal/store/builder.go +++ b/internal/store/builder.go @@ -18,74 +18,89 @@ package store import ( "context" + "fmt" "reflect" "sort" + "strconv" "strings" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" - autoscaling "k8s.io/api/autoscaling/v2beta1" + autoscaling "k8s.io/api/autoscaling/v2" batchv1 "k8s.io/api/batch/v1" - batchv1beta1 "k8s.io/api/batch/v1beta1" - certv1beta1 "k8s.io/api/certificates/v1beta1" + certv1 "k8s.io/api/certificates/v1" + coordinationv1 "k8s.io/api/coordination/v1" v1 "k8s.io/api/core/v1" - extensions "k8s.io/api/extensions/v1beta1" + discoveryv1 "k8s.io/api/discovery/v1" networkingv1 "k8s.io/api/networking/v1" - policy "k8s.io/api/policy/v1beta1" + policyv1 "k8s.io/api/policy/v1" + rbacv1 "k8s.io/api/rbac/v1" storagev1 "k8s.io/api/storage/v1" vpaautoscaling "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1beta2" vpaclientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" - "k8s.io/klog" - - "k8s.io/kube-state-metrics/pkg/listwatch" - "k8s.io/kube-state-metrics/pkg/metric" - metricsstore "k8s.io/kube-state-metrics/pkg/metrics_store" - "k8s.io/kube-state-metrics/pkg/options" - "k8s.io/kube-state-metrics/pkg/sharding" - "k8s.io/kube-state-metrics/pkg/watch" + "k8s.io/klog/v2" + + ksmtypes "k8s.io/kube-state-metrics/v2/pkg/builder/types" + "k8s.io/kube-state-metrics/v2/pkg/customresource" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" + metricsstore "k8s.io/kube-state-metrics/v2/pkg/metrics_store" + "k8s.io/kube-state-metrics/v2/pkg/options" + "k8s.io/kube-state-metrics/v2/pkg/sharding" + "k8s.io/kube-state-metrics/v2/pkg/watch" ) -type whiteBlackLister interface { - IsIncluded(string) bool - IsExcluded(string) bool -} +// Make sure the internal Builder implements the public BuilderInterface. +// New Builder methods should be added to the public BuilderInterface. +var _ ksmtypes.BuilderInterface = &Builder{} // Builder helps to build store. It follows the builder pattern // (https://en.wikipedia.org/wiki/Builder_pattern). type Builder struct { - kubeClient clientset.Interface - vpaClient vpaclientset.Interface - namespaces options.NamespaceList - ctx context.Context - enabledResources []string - whiteBlackList whiteBlackLister - metrics *watch.ListWatchMetrics - shard int32 - totalShards int + kubeClient clientset.Interface + customResourceClients map[string]interface{} + vpaClient vpaclientset.Interface + namespaces options.NamespaceList + // namespaceFilter is inside fieldSelectorFilter + fieldSelectorFilter string + ctx context.Context + enabledResources []string + familyGeneratorFilter generator.FamilyGeneratorFilter + listWatchMetrics *watch.ListWatchMetrics + shardingMetrics *sharding.Metrics + shard int32 + totalShards int + buildStoresFunc ksmtypes.BuildStoresFunc + buildCustomResourceStoresFunc ksmtypes.BuildCustomResourceStoresFunc + allowAnnotationsList map[string][]string + allowLabelsList map[string][]string + useAPIServerCache bool } // NewBuilder returns a new builder. -func NewBuilder() *Builder { return &Builder{} } +func NewBuilder() *Builder { + b := &Builder{} + return b +} // WithMetrics sets the metrics property of a Builder. -func (b *Builder) WithMetrics(r *prometheus.Registry) { - b.metrics = watch.NewListWatchMetrics(r) +func (b *Builder) WithMetrics(r prometheus.Registerer) { + b.listWatchMetrics = watch.NewListWatchMetrics(r) + b.shardingMetrics = sharding.NewShardingMetrics(r) } // WithEnabledResources sets the enabledResources property of a Builder. -func (b *Builder) WithEnabledResources(c []string) error { - for _, col := range c { - if !collectorExists(col) { - return errors.Errorf("collector %s does not exist. Available collectors: %s", col, strings.Join(availableCollectors(), ",")) +func (b *Builder) WithEnabledResources(r []string) error { + for _, col := range r { + if !resourceExists(col) { + return fmt.Errorf("resource %s does not exist. Available resources: %s", col, strings.Join(availableResources(), ",")) } } var copy []string - copy = append(copy, c...) + copy = append(copy, r...) sort.Strings(copy) @@ -93,15 +108,29 @@ func (b *Builder) WithEnabledResources(c []string) error { return nil } +// WithFieldSelectorFilter sets the fieldSelector property of a Builder. +func (b *Builder) WithFieldSelectorFilter(fieldSelectorFilter string) { + b.fieldSelectorFilter = fieldSelectorFilter +} + // WithNamespaces sets the namespaces property of a Builder. func (b *Builder) WithNamespaces(n options.NamespaceList) { b.namespaces = n } +// MergeFieldSelectors merges multiple fieldSelectors using AND operator. +func (b *Builder) MergeFieldSelectors(selectors []string) (string, error) { + return options.MergeFieldSelectors(selectors) +} + // WithSharding sets the shard and totalShards property of a Builder. func (b *Builder) WithSharding(shard int32, totalShards int) { b.shard = shard + labels := map[string]string{sharding.LabelOrdinal: strconv.Itoa(int(shard))} + b.shardingMetrics.Ordinal.Reset() + b.shardingMetrics.Ordinal.With(labels).Set(float64(shard)) b.totalShards = totalShards + b.shardingMetrics.Total.Set(float64(totalShards)) } // WithContext sets the ctx property of a Builder. @@ -119,72 +148,184 @@ func (b *Builder) WithVPAClient(c vpaclientset.Interface) { b.vpaClient = c } -// WithWhiteBlackList configures the white or blacklisted metric to be exposed -// by the store build by the Builder. -func (b *Builder) WithWhiteBlackList(l whiteBlackLister) { - b.whiteBlackList = l +// WithCustomResourceClients sets the customResourceClients property of a Builder. +func (b *Builder) WithCustomResourceClients(cs map[string]interface{}) { + b.customResourceClients = cs +} + +// WithUsingAPIServerCache configures whether using APIServer cache or not. +func (b *Builder) WithUsingAPIServerCache(u bool) { + b.useAPIServerCache = u +} + +// WithFamilyGeneratorFilter configures the family generator filter which decides which +// metrics are to be exposed by the store build by the Builder. +func (b *Builder) WithFamilyGeneratorFilter(l generator.FamilyGeneratorFilter) { + b.familyGeneratorFilter = l +} + +// WithGenerateStoresFunc configures a custom generate store function +func (b *Builder) WithGenerateStoresFunc(f ksmtypes.BuildStoresFunc) { + b.buildStoresFunc = f +} + +// WithGenerateCustomResourceStoresFunc configures a custom generate custom resource store function +func (b *Builder) WithGenerateCustomResourceStoresFunc(f ksmtypes.BuildCustomResourceStoresFunc) { + b.buildCustomResourceStoresFunc = f +} + +// DefaultGenerateStoresFunc returns default buildStores function +func (b *Builder) DefaultGenerateStoresFunc() ksmtypes.BuildStoresFunc { + return b.buildStores +} + +// DefaultGenerateCustomResourceStoresFunc returns default buildCustomResourceStores function +func (b *Builder) DefaultGenerateCustomResourceStoresFunc() ksmtypes.BuildCustomResourceStoresFunc { + return b.buildCustomResourceStores +} + +// WithCustomResourceStoreFactories returns configures a custom resource stores factory +func (b *Builder) WithCustomResourceStoreFactories(fs ...customresource.RegistryFactory) { + for i := range fs { + f := fs[i] + if _, ok := availableStores[f.Name()]; ok { + klog.InfoS("The internal resource store already exists and is overridden by a custom resource store with the same name, please make sure it meets your expectation", "registryName", f.Name()) + } + availableStores[f.Name()] = func(b *Builder) []cache.Store { + return b.buildCustomResourceStoresFunc( + f.Name(), + f.MetricFamilyGenerators(b.allowAnnotationsList[f.Name()], b.allowLabelsList[f.Name()]), + f.ExpectedType(), + f.ListWatch, + b.useAPIServerCache, + ) + } + } +} + +// WithAllowAnnotations configures which annotations can be returned for metrics +func (b *Builder) WithAllowAnnotations(annotations map[string][]string) { + if len(annotations) > 0 { + b.allowAnnotationsList = annotations + } +} + +// WithAllowLabels configures which labels can be returned for metrics +func (b *Builder) WithAllowLabels(labels map[string][]string) error { + if len(labels) > 0 { + for label := range labels { + if !resourceExists(label) && label != "*" { + return fmt.Errorf("resource %s does not exist. Available resources: %s", label, strings.Join(availableResources(), ",")) + } + } + b.allowLabelsList = labels + // "*" takes precedence over other specifications + if allowedLabels, ok := labels["*"]; ok { + m := make(map[string][]string) + for _, resource := range b.enabledResources { + m[resource] = allowedLabels + } + b.allowLabelsList = m + } + } + return nil } // Build initializes and registers all enabled stores. -func (b *Builder) Build() []*metricsstore.MetricsStore { - if b.whiteBlackList == nil { - panic("whiteBlackList should not be nil") +// It returns metrics writers which can be used to write out +// metrics from the stores. +func (b *Builder) Build() metricsstore.MetricsWriterList { + if b.familyGeneratorFilter == nil { + panic("familyGeneratorFilter should not be nil") } - stores := []*metricsstore.MetricsStore{} - activeStoreNames := []string{} + var metricsWriters metricsstore.MetricsWriterList + var activeStoreNames []string for _, c := range b.enabledResources { constructor, ok := availableStores[c] if ok { - store := constructor(b) + stores := cacheStoresToMetricStores(constructor(b)) activeStoreNames = append(activeStoreNames, c) - stores = append(stores, store) + metricsWriters = append(metricsWriters, metricsstore.NewMetricsWriter(stores...)) } } - klog.Infof("Active collectors: %s", strings.Join(activeStoreNames, ",")) + klog.InfoS("Active resources", "activeStoreNames", strings.Join(activeStoreNames, ",")) - return stores + return metricsWriters } -var availableStores = map[string]func(f *Builder) *metricsstore.MetricsStore{ - "certificatesigningrequests": func(b *Builder) *metricsstore.MetricsStore { return b.buildCsrStore() }, - "configmaps": func(b *Builder) *metricsstore.MetricsStore { return b.buildConfigMapStore() }, - "cronjobs": func(b *Builder) *metricsstore.MetricsStore { return b.buildCronJobStore() }, - "daemonsets": func(b *Builder) *metricsstore.MetricsStore { return b.buildDaemonSetStore() }, - "deployments": func(b *Builder) *metricsstore.MetricsStore { return b.buildDeploymentStore() }, - "endpoints": func(b *Builder) *metricsstore.MetricsStore { return b.buildEndpointsStore() }, - "horizontalpodautoscalers": func(b *Builder) *metricsstore.MetricsStore { return b.buildHPAStore() }, - "ingresses": func(b *Builder) *metricsstore.MetricsStore { return b.buildIngressStore() }, - "jobs": func(b *Builder) *metricsstore.MetricsStore { return b.buildJobStore() }, - "limitranges": func(b *Builder) *metricsstore.MetricsStore { return b.buildLimitRangeStore() }, - "mutatingwebhookconfigurations": func(b *Builder) *metricsstore.MetricsStore { return b.buildMutatingWebhookConfigurationStore() }, - "namespaces": func(b *Builder) *metricsstore.MetricsStore { return b.buildNamespaceStore() }, - "networkpolicies": func(b *Builder) *metricsstore.MetricsStore { return b.buildNetworkPolicyStore() }, - "nodes": func(b *Builder) *metricsstore.MetricsStore { return b.buildNodeStore() }, - "persistentvolumeclaims": func(b *Builder) *metricsstore.MetricsStore { return b.buildPersistentVolumeClaimStore() }, - "persistentvolumes": func(b *Builder) *metricsstore.MetricsStore { return b.buildPersistentVolumeStore() }, - "poddisruptionbudgets": func(b *Builder) *metricsstore.MetricsStore { return b.buildPodDisruptionBudgetStore() }, - "pods": func(b *Builder) *metricsstore.MetricsStore { return b.buildPodStore() }, - "replicasets": func(b *Builder) *metricsstore.MetricsStore { return b.buildReplicaSetStore() }, - "replicationcontrollers": func(b *Builder) *metricsstore.MetricsStore { return b.buildReplicationControllerStore() }, - "resourcequotas": func(b *Builder) *metricsstore.MetricsStore { return b.buildResourceQuotaStore() }, - "secrets": func(b *Builder) *metricsstore.MetricsStore { return b.buildSecretStore() }, - "services": func(b *Builder) *metricsstore.MetricsStore { return b.buildServiceStore() }, - "statefulsets": func(b *Builder) *metricsstore.MetricsStore { return b.buildStatefulSetStore() }, - "storageclasses": func(b *Builder) *metricsstore.MetricsStore { return b.buildStorageClassStore() }, - "validatingwebhookconfigurations": func(b *Builder) *metricsstore.MetricsStore { return b.buildValidatingWebhookConfigurationStore() }, - "volumeattachments": func(b *Builder) *metricsstore.MetricsStore { return b.buildVolumeAttachmentStore() }, - "verticalpodautoscalers": func(b *Builder) *metricsstore.MetricsStore { return b.buildVPAStore() }, -} - -func collectorExists(name string) bool { +// BuildStores initializes and registers all enabled stores. +// It returns metric stores which can be used to consume +// the generated metrics from the stores. +func (b *Builder) BuildStores() [][]cache.Store { + if b.familyGeneratorFilter == nil { + panic("familyGeneratorFilter should not be nil") + } + + var allStores [][]cache.Store + var activeStoreNames []string + + for _, c := range b.enabledResources { + constructor, ok := availableStores[c] + if ok { + stores := constructor(b) + activeStoreNames = append(activeStoreNames, c) + allStores = append(allStores, stores) + } + } + + klog.InfoS("Active resources", "activeStoreNames", strings.Join(activeStoreNames, ",")) + + return allStores +} + +var availableStores = map[string]func(f *Builder) []cache.Store{ + "certificatesigningrequests": func(b *Builder) []cache.Store { return b.buildCsrStores() }, + "clusterroles": func(b *Builder) []cache.Store { return b.buildClusterRoleStores() }, + "configmaps": func(b *Builder) []cache.Store { return b.buildConfigMapStores() }, + "clusterrolebindings": func(b *Builder) []cache.Store { return b.buildClusterRoleBindingStores() }, + "cronjobs": func(b *Builder) []cache.Store { return b.buildCronJobStores() }, + "daemonsets": func(b *Builder) []cache.Store { return b.buildDaemonSetStores() }, + "deployments": func(b *Builder) []cache.Store { return b.buildDeploymentStores() }, + "endpoints": func(b *Builder) []cache.Store { return b.buildEndpointsStores() }, + "endpointslices": func(b *Builder) []cache.Store { return b.buildEndpointSlicesStores() }, + "horizontalpodautoscalers": func(b *Builder) []cache.Store { return b.buildHPAStores() }, + "ingresses": func(b *Builder) []cache.Store { return b.buildIngressStores() }, + "ingressclasses": func(b *Builder) []cache.Store { return b.buildIngressClassStores() }, + "jobs": func(b *Builder) []cache.Store { return b.buildJobStores() }, + "leases": func(b *Builder) []cache.Store { return b.buildLeasesStores() }, + "limitranges": func(b *Builder) []cache.Store { return b.buildLimitRangeStores() }, + "mutatingwebhookconfigurations": func(b *Builder) []cache.Store { return b.buildMutatingWebhookConfigurationStores() }, + "namespaces": func(b *Builder) []cache.Store { return b.buildNamespaceStores() }, + "networkpolicies": func(b *Builder) []cache.Store { return b.buildNetworkPolicyStores() }, + "nodes": func(b *Builder) []cache.Store { return b.buildNodeStores() }, + "persistentvolumeclaims": func(b *Builder) []cache.Store { return b.buildPersistentVolumeClaimStores() }, + "persistentvolumes": func(b *Builder) []cache.Store { return b.buildPersistentVolumeStores() }, + "poddisruptionbudgets": func(b *Builder) []cache.Store { return b.buildPodDisruptionBudgetStores() }, + "pods": func(b *Builder) []cache.Store { return b.buildPodStores() }, + "replicasets": func(b *Builder) []cache.Store { return b.buildReplicaSetStores() }, + "replicationcontrollers": func(b *Builder) []cache.Store { return b.buildReplicationControllerStores() }, + "resourcequotas": func(b *Builder) []cache.Store { return b.buildResourceQuotaStores() }, + "roles": func(b *Builder) []cache.Store { return b.buildRoleStores() }, + "rolebindings": func(b *Builder) []cache.Store { return b.buildRoleBindingStores() }, + "secrets": func(b *Builder) []cache.Store { return b.buildSecretStores() }, + "serviceaccounts": func(b *Builder) []cache.Store { return b.buildServiceAccountStores() }, + "services": func(b *Builder) []cache.Store { return b.buildServiceStores() }, + "statefulsets": func(b *Builder) []cache.Store { return b.buildStatefulSetStores() }, + "storageclasses": func(b *Builder) []cache.Store { return b.buildStorageClassStores() }, + "validatingwebhookconfigurations": func(b *Builder) []cache.Store { return b.buildValidatingWebhookConfigurationStores() }, + "volumeattachments": func(b *Builder) []cache.Store { return b.buildVolumeAttachmentStores() }, + "verticalpodautoscalers": func(b *Builder) []cache.Store { return b.buildVPAStores() }, +} + +func resourceExists(name string) bool { _, ok := availableStores[name] return ok } -func availableCollectors() []string { +func availableResources() []string { c := []string{} for name := range availableStores { c = append(c, name) @@ -192,147 +333,254 @@ func availableCollectors() []string { return c } -func (b *Builder) buildConfigMapStore() *metricsstore.MetricsStore { - return b.buildStore(configMapMetricFamilies, &v1.ConfigMap{}, createConfigMapListWatch) +func (b *Builder) buildConfigMapStores() []cache.Store { + return b.buildStoresFunc(configMapMetricFamilies(b.allowAnnotationsList["configmaps"], b.allowLabelsList["configmaps"]), &v1.ConfigMap{}, createConfigMapListWatch, b.useAPIServerCache) +} + +func (b *Builder) buildCronJobStores() []cache.Store { + return b.buildStoresFunc(cronJobMetricFamilies(b.allowAnnotationsList["cronjobs"], b.allowLabelsList["cronjobs"]), &batchv1.CronJob{}, createCronJobListWatch, b.useAPIServerCache) +} + +func (b *Builder) buildDaemonSetStores() []cache.Store { + return b.buildStoresFunc(daemonSetMetricFamilies(b.allowAnnotationsList["daemonsets"], b.allowLabelsList["daemonsets"]), &appsv1.DaemonSet{}, createDaemonSetListWatch, b.useAPIServerCache) +} + +func (b *Builder) buildDeploymentStores() []cache.Store { + return b.buildStoresFunc(deploymentMetricFamilies(b.allowAnnotationsList["deployments"], b.allowLabelsList["deployments"]), &appsv1.Deployment{}, createDeploymentListWatch, b.useAPIServerCache) +} + +func (b *Builder) buildEndpointsStores() []cache.Store { + return b.buildStoresFunc(endpointMetricFamilies(b.allowAnnotationsList["endpoints"], b.allowLabelsList["endpoints"]), &v1.Endpoints{}, createEndpointsListWatch, b.useAPIServerCache) +} + +func (b *Builder) buildEndpointSlicesStores() []cache.Store { + return b.buildStoresFunc(endpointSliceMetricFamilies(b.allowAnnotationsList["endpointslices"], b.allowLabelsList["endpointslices"]), &discoveryv1.EndpointSlice{}, createEndpointSliceListWatch, b.useAPIServerCache) +} + +func (b *Builder) buildHPAStores() []cache.Store { + return b.buildStoresFunc(hpaMetricFamilies(b.allowAnnotationsList["horizontalpodautoscalers"], b.allowLabelsList["horizontalpodautoscalers"]), &autoscaling.HorizontalPodAutoscaler{}, createHPAListWatch, b.useAPIServerCache) +} + +func (b *Builder) buildIngressStores() []cache.Store { + return b.buildStoresFunc(ingressMetricFamilies(b.allowAnnotationsList["ingresses"], b.allowLabelsList["ingresses"]), &networkingv1.Ingress{}, createIngressListWatch, b.useAPIServerCache) +} + +func (b *Builder) buildJobStores() []cache.Store { + return b.buildStoresFunc(jobMetricFamilies(b.allowAnnotationsList["jobs"], b.allowLabelsList["jobs"]), &batchv1.Job{}, createJobListWatch, b.useAPIServerCache) } -func (b *Builder) buildCronJobStore() *metricsstore.MetricsStore { - return b.buildStore(cronJobMetricFamilies, &batchv1beta1.CronJob{}, createCronJobListWatch) +func (b *Builder) buildLimitRangeStores() []cache.Store { + return b.buildStoresFunc(limitRangeMetricFamilies, &v1.LimitRange{}, createLimitRangeListWatch, b.useAPIServerCache) } -func (b *Builder) buildDaemonSetStore() *metricsstore.MetricsStore { - return b.buildStore(daemonSetMetricFamilies, &appsv1.DaemonSet{}, createDaemonSetListWatch) +func (b *Builder) buildMutatingWebhookConfigurationStores() []cache.Store { + return b.buildStoresFunc(mutatingWebhookConfigurationMetricFamilies, &admissionregistrationv1.MutatingWebhookConfiguration{}, createMutatingWebhookConfigurationListWatch, b.useAPIServerCache) } -func (b *Builder) buildDeploymentStore() *metricsstore.MetricsStore { - return b.buildStore(deploymentMetricFamilies, &appsv1.Deployment{}, createDeploymentListWatch) +func (b *Builder) buildNamespaceStores() []cache.Store { + return b.buildStoresFunc(namespaceMetricFamilies(b.allowAnnotationsList["namespaces"], b.allowLabelsList["namespaces"]), &v1.Namespace{}, createNamespaceListWatch, b.useAPIServerCache) } -func (b *Builder) buildEndpointsStore() *metricsstore.MetricsStore { - return b.buildStore(endpointMetricFamilies, &v1.Endpoints{}, createEndpointsListWatch) +func (b *Builder) buildNetworkPolicyStores() []cache.Store { + return b.buildStoresFunc(networkPolicyMetricFamilies(b.allowAnnotationsList["networkpolicies"], b.allowLabelsList["networkpolicies"]), &networkingv1.NetworkPolicy{}, createNetworkPolicyListWatch, b.useAPIServerCache) } -func (b *Builder) buildHPAStore() *metricsstore.MetricsStore { - return b.buildStore(hpaMetricFamilies, &autoscaling.HorizontalPodAutoscaler{}, createHPAListWatch) +func (b *Builder) buildNodeStores() []cache.Store { + return b.buildStoresFunc(nodeMetricFamilies(b.allowAnnotationsList["nodes"], b.allowLabelsList["nodes"]), &v1.Node{}, createNodeListWatch, b.useAPIServerCache) } -func (b *Builder) buildIngressStore() *metricsstore.MetricsStore { - return b.buildStore(ingressMetricFamilies, &extensions.Ingress{}, createIngressListWatch) +func (b *Builder) buildPersistentVolumeClaimStores() []cache.Store { + return b.buildStoresFunc(persistentVolumeClaimMetricFamilies(b.allowAnnotationsList["persistentvolumeclaims"], b.allowLabelsList["persistentvolumeclaims"]), &v1.PersistentVolumeClaim{}, createPersistentVolumeClaimListWatch, b.useAPIServerCache) } -func (b *Builder) buildJobStore() *metricsstore.MetricsStore { - return b.buildStore(jobMetricFamilies, &batchv1.Job{}, createJobListWatch) +func (b *Builder) buildPersistentVolumeStores() []cache.Store { + return b.buildStoresFunc(persistentVolumeMetricFamilies(b.allowAnnotationsList["persistentvolumes"], b.allowLabelsList["persistentvolumes"]), &v1.PersistentVolume{}, createPersistentVolumeListWatch, b.useAPIServerCache) } -func (b *Builder) buildLimitRangeStore() *metricsstore.MetricsStore { - return b.buildStore(limitRangeMetricFamilies, &v1.LimitRange{}, createLimitRangeListWatch) +func (b *Builder) buildPodDisruptionBudgetStores() []cache.Store { + return b.buildStoresFunc(podDisruptionBudgetMetricFamilies(b.allowAnnotationsList["poddisruptionbudgets"], b.allowLabelsList["poddisruptionbudgets"]), &policyv1.PodDisruptionBudget{}, createPodDisruptionBudgetListWatch, b.useAPIServerCache) } -func (b *Builder) buildMutatingWebhookConfigurationStore() *metricsstore.MetricsStore { - return b.buildStore(mutatingWebhookConfigurationMetricFamilies, &admissionregistrationv1.MutatingWebhookConfiguration{}, createMutatingWebhookConfigurationListWatch) +func (b *Builder) buildReplicaSetStores() []cache.Store { + return b.buildStoresFunc(replicaSetMetricFamilies(b.allowAnnotationsList["replicasets"], b.allowLabelsList["replicasets"]), &appsv1.ReplicaSet{}, createReplicaSetListWatch, b.useAPIServerCache) } -func (b *Builder) buildNamespaceStore() *metricsstore.MetricsStore { - return b.buildStore(namespaceMetricFamilies, &v1.Namespace{}, createNamespaceListWatch) +func (b *Builder) buildReplicationControllerStores() []cache.Store { + return b.buildStoresFunc(replicationControllerMetricFamilies, &v1.ReplicationController{}, createReplicationControllerListWatch, b.useAPIServerCache) } -func (b *Builder) buildNetworkPolicyStore() *metricsstore.MetricsStore { - return b.buildStore(networkpolicyMetricFamilies, &networkingv1.NetworkPolicy{}, createNetworkPolicyListWatch) +func (b *Builder) buildResourceQuotaStores() []cache.Store { + return b.buildStoresFunc(resourceQuotaMetricFamilies, &v1.ResourceQuota{}, createResourceQuotaListWatch, b.useAPIServerCache) } -func (b *Builder) buildNodeStore() *metricsstore.MetricsStore { - return b.buildStore(nodeMetricFamilies, &v1.Node{}, createNodeListWatch) +func (b *Builder) buildSecretStores() []cache.Store { + return b.buildStoresFunc(secretMetricFamilies(b.allowAnnotationsList["secrets"], b.allowLabelsList["secrets"]), &v1.Secret{}, createSecretListWatch, b.useAPIServerCache) } -func (b *Builder) buildPersistentVolumeClaimStore() *metricsstore.MetricsStore { - return b.buildStore(persistentVolumeClaimMetricFamilies, &v1.PersistentVolumeClaim{}, createPersistentVolumeClaimListWatch) +func (b *Builder) buildServiceAccountStores() []cache.Store { + return b.buildStoresFunc(serviceAccountMetricFamilies(b.allowAnnotationsList["serviceaccounts"], b.allowLabelsList["serviceaccounts"]), &v1.ServiceAccount{}, createServiceAccountListWatch, b.useAPIServerCache) } -func (b *Builder) buildPersistentVolumeStore() *metricsstore.MetricsStore { - return b.buildStore(persistentVolumeMetricFamilies, &v1.PersistentVolume{}, createPersistentVolumeListWatch) +func (b *Builder) buildServiceStores() []cache.Store { + return b.buildStoresFunc(serviceMetricFamilies(b.allowAnnotationsList["services"], b.allowLabelsList["services"]), &v1.Service{}, createServiceListWatch, b.useAPIServerCache) } -func (b *Builder) buildPodDisruptionBudgetStore() *metricsstore.MetricsStore { - return b.buildStore(podDisruptionBudgetMetricFamilies, &policy.PodDisruptionBudget{}, createPodDisruptionBudgetListWatch) +func (b *Builder) buildStatefulSetStores() []cache.Store { + return b.buildStoresFunc(statefulSetMetricFamilies(b.allowAnnotationsList["statefulsets"], b.allowLabelsList["statefulsets"]), &appsv1.StatefulSet{}, createStatefulSetListWatch, b.useAPIServerCache) } -func (b *Builder) buildReplicaSetStore() *metricsstore.MetricsStore { - return b.buildStore(replicaSetMetricFamilies, &appsv1.ReplicaSet{}, createReplicaSetListWatch) +func (b *Builder) buildStorageClassStores() []cache.Store { + return b.buildStoresFunc(storageClassMetricFamilies(b.allowAnnotationsList["storageclasses"], b.allowLabelsList["storageclasses"]), &storagev1.StorageClass{}, createStorageClassListWatch, b.useAPIServerCache) } -func (b *Builder) buildReplicationControllerStore() *metricsstore.MetricsStore { - return b.buildStore(replicationControllerMetricFamilies, &v1.ReplicationController{}, createReplicationControllerListWatch) +func (b *Builder) buildPodStores() []cache.Store { + return b.buildStoresFunc(podMetricFamilies(b.allowAnnotationsList["pods"], b.allowLabelsList["pods"]), &v1.Pod{}, createPodListWatch, b.useAPIServerCache) } -func (b *Builder) buildResourceQuotaStore() *metricsstore.MetricsStore { - return b.buildStore(resourceQuotaMetricFamilies, &v1.ResourceQuota{}, createResourceQuotaListWatch) +func (b *Builder) buildCsrStores() []cache.Store { + return b.buildStoresFunc(csrMetricFamilies(b.allowAnnotationsList["certificatesigningrequests"], b.allowLabelsList["certificatesigningrequests"]), &certv1.CertificateSigningRequest{}, createCSRListWatch, b.useAPIServerCache) } -func (b *Builder) buildSecretStore() *metricsstore.MetricsStore { - return b.buildStore(secretMetricFamilies, &v1.Secret{}, createSecretListWatch) +func (b *Builder) buildValidatingWebhookConfigurationStores() []cache.Store { + return b.buildStoresFunc(validatingWebhookConfigurationMetricFamilies, &admissionregistrationv1.ValidatingWebhookConfiguration{}, createValidatingWebhookConfigurationListWatch, b.useAPIServerCache) } -func (b *Builder) buildServiceStore() *metricsstore.MetricsStore { - return b.buildStore(serviceMetricFamilies, &v1.Service{}, createServiceListWatch) +func (b *Builder) buildVolumeAttachmentStores() []cache.Store { + return b.buildStoresFunc(volumeAttachmentMetricFamilies, &storagev1.VolumeAttachment{}, createVolumeAttachmentListWatch, b.useAPIServerCache) } -func (b *Builder) buildStatefulSetStore() *metricsstore.MetricsStore { - return b.buildStore(statefulSetMetricFamilies, &appsv1.StatefulSet{}, createStatefulSetListWatch) +func (b *Builder) buildVPAStores() []cache.Store { + return b.buildStoresFunc(vpaMetricFamilies(b.allowAnnotationsList["verticalpodautoscalers"], b.allowLabelsList["verticalpodautoscalers"]), &vpaautoscaling.VerticalPodAutoscaler{}, createVPAListWatchFunc(b.vpaClient), b.useAPIServerCache) } -func (b *Builder) buildStorageClassStore() *metricsstore.MetricsStore { - return b.buildStore(storageClassMetricFamilies, &storagev1.StorageClass{}, createStorageClassListWatch) +func (b *Builder) buildLeasesStores() []cache.Store { + return b.buildStoresFunc(leaseMetricFamilies, &coordinationv1.Lease{}, createLeaseListWatch, b.useAPIServerCache) } -func (b *Builder) buildPodStore() *metricsstore.MetricsStore { - return b.buildStore(podMetricFamilies, &v1.Pod{}, createPodListWatch) +func (b *Builder) buildClusterRoleStores() []cache.Store { + return b.buildStoresFunc(clusterRoleMetricFamilies(b.allowAnnotationsList["clusterroles"], b.allowLabelsList["clusterroles"]), &rbacv1.ClusterRole{}, createClusterRoleListWatch, b.useAPIServerCache) } -func (b *Builder) buildCsrStore() *metricsstore.MetricsStore { - return b.buildStore(csrMetricFamilies, &certv1beta1.CertificateSigningRequest{}, createCSRListWatch) +func (b *Builder) buildRoleStores() []cache.Store { + return b.buildStoresFunc(roleMetricFamilies(b.allowAnnotationsList["roles"], b.allowLabelsList["roles"]), &rbacv1.Role{}, createRoleListWatch, b.useAPIServerCache) } -func (b *Builder) buildValidatingWebhookConfigurationStore() *metricsstore.MetricsStore { - return b.buildStore(validatingWebhookConfigurationMetricFamilies, &admissionregistrationv1.ValidatingWebhookConfiguration{}, createValidatingWebhookConfigurationListWatch) +func (b *Builder) buildClusterRoleBindingStores() []cache.Store { + return b.buildStoresFunc(clusterRoleBindingMetricFamilies(b.allowAnnotationsList["clusterrolebindings"], b.allowLabelsList["clusterrolebindings"]), &rbacv1.ClusterRoleBinding{}, createClusterRoleBindingListWatch, b.useAPIServerCache) } -func (b *Builder) buildVolumeAttachmentStore() *metricsstore.MetricsStore { - return b.buildStore(volumeAttachmentMetricFamilies, &storagev1.VolumeAttachment{}, createVolumeAttachmentListWatch) +func (b *Builder) buildRoleBindingStores() []cache.Store { + return b.buildStoresFunc(roleBindingMetricFamilies(b.allowAnnotationsList["rolebindings"], b.allowLabelsList["rolebindings"]), &rbacv1.RoleBinding{}, createRoleBindingListWatch, b.useAPIServerCache) } -func (b *Builder) buildVPAStore() *metricsstore.MetricsStore { - return b.buildStore(vpaMetricFamilies, &vpaautoscaling.VerticalPodAutoscaler{}, createVPAListWatchFunc(b.vpaClient)) +func (b *Builder) buildIngressClassStores() []cache.Store { + return b.buildStoresFunc(ingressClassMetricFamilies(b.allowAnnotationsList["ingressclasses"], b.allowLabelsList["ingressclasses"]), &networkingv1.IngressClass{}, createIngressClassListWatch, b.useAPIServerCache) } -func (b *Builder) buildStore( - metricFamilies []metric.FamilyGenerator, +func (b *Builder) buildStores( + metricFamilies []generator.FamilyGenerator, expectedType interface{}, - listWatchFunc func(kubeClient clientset.Interface, ns string) cache.ListerWatcher, -) *metricsstore.MetricsStore { - filteredMetricFamilies := metric.FilterMetricFamilies(b.whiteBlackList, metricFamilies) - composedMetricGenFuncs := metric.ComposeMetricGenFuncs(filteredMetricFamilies) + listWatchFunc func(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher, + useAPIServerCache bool, +) []cache.Store { + metricFamilies = generator.FilterFamilyGenerators(b.familyGeneratorFilter, metricFamilies) + composedMetricGenFuncs := generator.ComposeMetricGenFuncs(metricFamilies) + familyHeaders := generator.ExtractMetricFamilyHeaders(metricFamilies) + + if b.namespaces.IsAllNamespaces() { + store := metricsstore.NewMetricsStore( + familyHeaders, + composedMetricGenFuncs, + ) + if b.fieldSelectorFilter != "" { + klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter) + } + listWatcher := listWatchFunc(b.kubeClient, v1.NamespaceAll, b.fieldSelectorFilter) + b.startReflector(expectedType, store, listWatcher, useAPIServerCache) + return []cache.Store{store} + } + + stores := make([]cache.Store, 0, len(b.namespaces)) + for _, ns := range b.namespaces { + store := metricsstore.NewMetricsStore( + familyHeaders, + composedMetricGenFuncs, + ) + if b.fieldSelectorFilter != "" { + klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter) + } + listWatcher := listWatchFunc(b.kubeClient, ns, b.fieldSelectorFilter) + b.startReflector(expectedType, store, listWatcher, useAPIServerCache) + stores = append(stores, store) + } - familyHeaders := metric.ExtractMetricFamilyHeaders(filteredMetricFamilies) + return stores +} - store := metricsstore.NewMetricsStore( - familyHeaders, - composedMetricGenFuncs, - ) - b.reflectorPerNamespace(expectedType, store, listWatchFunc) +// TODO(Garrybest): Merge `buildStores` and `buildCustomResourceStores` +func (b *Builder) buildCustomResourceStores(resourceName string, + metricFamilies []generator.FamilyGenerator, + expectedType interface{}, + listWatchFunc func(customResourceClient interface{}, ns string, fieldSelector string) cache.ListerWatcher, + useAPIServerCache bool, +) []cache.Store { + metricFamilies = generator.FilterFamilyGenerators(b.familyGeneratorFilter, metricFamilies) + composedMetricGenFuncs := generator.ComposeMetricGenFuncs(metricFamilies) + familyHeaders := generator.ExtractMetricFamilyHeaders(metricFamilies) + + customResourceClient, ok := b.customResourceClients[resourceName] + if !ok { + klog.InfoS("Custom resource client does not exist", "resourceName", resourceName) + return []cache.Store{} + } - return store + if b.namespaces.IsAllNamespaces() { + store := metricsstore.NewMetricsStore( + familyHeaders, + composedMetricGenFuncs, + ) + if b.fieldSelectorFilter != "" { + klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter) + } + listWatcher := listWatchFunc(customResourceClient, v1.NamespaceAll, b.fieldSelectorFilter) + b.startReflector(expectedType, store, listWatcher, useAPIServerCache) + return []cache.Store{store} + } + + stores := make([]cache.Store, 0, len(b.namespaces)) + for _, ns := range b.namespaces { + store := metricsstore.NewMetricsStore( + familyHeaders, + composedMetricGenFuncs, + ) + klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter) + listWatcher := listWatchFunc(customResourceClient, ns, b.fieldSelectorFilter) + b.startReflector(expectedType, store, listWatcher, useAPIServerCache) + stores = append(stores, store) + } + + return stores } -// reflectorPerNamespace creates a Kubernetes client-go reflector with the given -// listWatchFunc for each given namespace and registers it with the given store. -func (b *Builder) reflectorPerNamespace( +// startReflector starts a Kubernetes client-go reflector with the given +// listWatcher and registers it with the given store. +func (b *Builder) startReflector( expectedType interface{}, store cache.Store, - listWatchFunc func(kubeClient clientset.Interface, ns string) cache.ListerWatcher, + listWatcher cache.ListerWatcher, + useAPIServerCache bool, ) { - lwf := func(ns string) cache.ListerWatcher { return listWatchFunc(b.kubeClient, ns) } - lw := listwatch.MultiNamespaceListerWatcher(b.namespaces, nil, lwf) - instrumentedListWatch := watch.NewInstrumentedListerWatcher(lw, b.metrics, reflect.TypeOf(expectedType).String()) + instrumentedListWatch := watch.NewInstrumentedListerWatcher(listWatcher, b.listWatchMetrics, reflect.TypeOf(expectedType).String(), useAPIServerCache) reflector := cache.NewReflector(sharding.NewShardedListWatch(b.shard, b.totalShards, instrumentedListWatch), expectedType, store, 0) go reflector.Run(b.ctx.Done()) } + +// cacheStoresToMetricStores converts []cache.Store into []*metricsstore.MetricsStore +func cacheStoresToMetricStores(cStores []cache.Store) []*metricsstore.MetricsStore { + mStores := make([]*metricsstore.MetricsStore, 0, len(cStores)) + for _, store := range cStores { + mStores = append(mStores, store.(*metricsstore.MetricsStore)) + } + + return mStores +} diff --git a/internal/store/builder_test.go b/internal/store/builder_test.go new file mode 100644 index 0000000000..758781d6f7 --- /dev/null +++ b/internal/store/builder_test.go @@ -0,0 +1,115 @@ +/* +Copyright 2022 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "reflect" + "testing" + + "k8s.io/kube-state-metrics/v2/pkg/options" +) + +type LabelsAllowList options.LabelsAllowList + +type expectedError struct { + expectedResourceError bool + expectedLabelError bool + expectedNotEqual bool +} + +func TestWithAllowLabels(t *testing.T) { + tests := []struct { + Desc string + LabelsAllowlist map[string][]string + EnabledResources []string + Wanted LabelsAllowList + err expectedError + }{ + { + Desc: "wildcard key-value as the only element", + LabelsAllowlist: map[string][]string{"*": {"*"}}, + EnabledResources: []string{"cronjobs", "pods", "deployments"}, + Wanted: LabelsAllowList(map[string][]string{ + "deployments": {"*"}, + "pods": {"*"}, + "cronjobs": {"*"}, + }), + }, + { + Desc: "wildcard key-value as not the only element", + LabelsAllowlist: map[string][]string{"*": {"*"}, "pods": {"*"}, "cronjobs": {"*"}}, + EnabledResources: []string{"cronjobs", "pods", "deployments"}, + Wanted: LabelsAllowList(map[string][]string{ + "deployments": {"*"}, + "pods": {"*"}, + "cronjobs": {"*"}, + }), + }, + { + Desc: "wildcard key-value as not the only element, with resource mismatch", + LabelsAllowlist: map[string][]string{"*": {"*"}, "pods": {"*"}, "cronjobs": {"*"}, "configmaps": {"*"}}, + EnabledResources: []string{"cronjobs", "pods", "deployments"}, + Wanted: LabelsAllowList{}, + err: expectedError{ + expectedNotEqual: true, + }, + }, + { + Desc: "wildcard key-value as not the only element, with other mutually-exclusive keys", + LabelsAllowlist: map[string][]string{"*": {"*"}, "foo": {"*"}, "bar": {"*"}, "cronjobs": {"*"}}, + EnabledResources: []string{"cronjobs", "pods", "deployments"}, + Wanted: LabelsAllowList(nil), + err: expectedError{ + expectedLabelError: true, + }, + }, + { + Desc: "wildcard key-value as not the only element, with other resources that do not exist", + LabelsAllowlist: map[string][]string{"*": {"*"}, "cronjobs": {"*"}}, + EnabledResources: []string{"cronjobs", "pods", "deployments", "foo", "bar"}, + Wanted: LabelsAllowList{}, + err: expectedError{ + expectedResourceError: true, + }, + }, + } + + for _, test := range tests { + b := NewBuilder() + + // Set the enabled resources. + err := b.WithEnabledResources(test.EnabledResources) + if err != nil && !test.err.expectedResourceError { + t.Log("Did not expect error while setting resources (--resources).") + t.Errorf("Test error for Desc: %s. Got Error: %v", test.Desc, err) + } + + // Resolve the allow list. + err = b.WithAllowLabels(test.LabelsAllowlist) + if err != nil && !test.err.expectedLabelError { + t.Log("Did not expect error while parsing allow list labels (--metric-labels-allowlist).") + t.Errorf("Test error for Desc: %s. Got Error: %v", test.Desc, err) + } + resolvedAllowLabels := LabelsAllowList(b.allowLabelsList) + + // Evaluate. + if !reflect.DeepEqual(resolvedAllowLabels, test.Wanted) && !test.err.expectedNotEqual { + t.Log("Expected maps to be equal.") + t.Errorf("Test error for Desc: %s\n Want: \n%+v\n Got: \n%#+v", test.Desc, test.Wanted, resolvedAllowLabels) + } + } +} diff --git a/internal/store/certificatesigningrequest.go b/internal/store/certificatesigningrequest.go index 7c5ab5ee61..5a4a1b926f 100644 --- a/internal/store/certificatesigningrequest.go +++ b/internal/store/certificatesigningrequest.go @@ -17,9 +17,14 @@ limitations under the License. package store import ( - "k8s.io/kube-state-metrics/pkg/metric" + "context" - certv1beta1 "k8s.io/api/certificates/v1beta1" + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" + + certv1 "k8s.io/api/certificates/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" @@ -28,17 +33,42 @@ import ( ) var ( + descCSRAnnotationsName = "kube_certificatesigningrequest_annotations" + descCSRAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." descCSRLabelsName = "kube_certificatesigningrequest_labels" descCSRLabelsHelp = "Kubernetes labels converted to Prometheus labels." - descCSRLabelsDefaultLabels = []string{"certificatesigningrequest"} + descCSRLabelsDefaultLabels = []string{"certificatesigningrequest", "signer_name"} +) - csrMetricFamilies = []metric.FamilyGenerator{ - { - Name: descCSRLabelsName, - Type: metric.Gauge, - Help: descCSRLabelsHelp, - GenerateFunc: wrapCSRFunc(func(j *certv1beta1.CertificateSigningRequest) *metric.Family { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(j.Labels) +func csrMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + descCSRAnnotationsName, + descCSRAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapCSRFunc(func(j *certv1.CertificateSigningRequest) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", j.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descCSRLabelsName, + descCSRLabelsHelp, + metric.Gauge, + basemetrics.STABLE, + "", + wrapCSRFunc(func(j *certv1.CertificateSigningRequest) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", j.Labels, allowLabelsList) return &metric.Family{ Metrics: []*metric.Metric{ { @@ -49,12 +79,14 @@ var ( }, } }), - }, - { - Name: "kube_certificatesigningrequest_created", - Type: metric.Gauge, - Help: "Unix creation timestamp", - GenerateFunc: wrapCSRFunc(func(csr *certv1beta1.CertificateSigningRequest) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_certificatesigningrequest_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.STABLE, + "", + wrapCSRFunc(func(csr *certv1.CertificateSigningRequest) *metric.Family { ms := []*metric.Metric{} if !csr.CreationTimestamp.IsZero() { ms = append(ms, &metric.Metric{ @@ -68,22 +100,26 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_certificatesigningrequest_condition", - Type: metric.Gauge, - Help: "The number of each certificatesigningrequest condition", - GenerateFunc: wrapCSRFunc(func(csr *certv1beta1.CertificateSigningRequest) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_certificatesigningrequest_condition", + "The number of each certificatesigningrequest condition", + metric.Gauge, + basemetrics.STABLE, + "", + wrapCSRFunc(func(csr *certv1.CertificateSigningRequest) *metric.Family { return &metric.Family{ Metrics: addCSRConditionMetrics(csr.Status), } }), - }, - { - Name: "kube_certificatesigningrequest_cert_length", - Type: metric.Gauge, - Help: "Length of the issued cert", - GenerateFunc: wrapCSRFunc(func(csr *certv1beta1.CertificateSigningRequest) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_certificatesigningrequest_cert_length", + "Length of the issued cert", + metric.Gauge, + basemetrics.STABLE, + "", + wrapCSRFunc(func(csr *certv1.CertificateSigningRequest) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -94,45 +130,43 @@ var ( }, } }), - }, + ), } -) +} -func wrapCSRFunc(f func(*certv1beta1.CertificateSigningRequest) *metric.Family) func(interface{}) *metric.Family { +func wrapCSRFunc(f func(*certv1.CertificateSigningRequest) *metric.Family) func(interface{}) *metric.Family { return func(obj interface{}) *metric.Family { - csr := obj.(*certv1beta1.CertificateSigningRequest) - + csr := obj.(*certv1.CertificateSigningRequest) metricFamily := f(csr) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descCSRLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{csr.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descCSRLabelsDefaultLabels, []string{csr.Name, csr.Spec.SignerName}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createCSRListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createCSRListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.CertificatesV1beta1().CertificateSigningRequests().List(opts) + return kubeClient.CertificatesV1().CertificateSigningRequests().List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.CertificatesV1beta1().CertificateSigningRequests().Watch(opts) + return kubeClient.CertificatesV1().CertificateSigningRequests().Watch(context.TODO(), opts) }, } } // addCSRConditionMetrics generates one metric for each possible csr condition status -func addCSRConditionMetrics(cs certv1beta1.CertificateSigningRequestStatus) []*metric.Metric { +func addCSRConditionMetrics(cs certv1.CertificateSigningRequestStatus) []*metric.Metric { cApproved := 0 cDenied := 0 for _, s := range cs.Conditions { - if s.Type == certv1beta1.CertificateApproved { + if s.Type == certv1.CertificateApproved { cApproved++ } - if s.Type == certv1beta1.CertificateDenied { + if s.Type == certv1.CertificateDenied { cDenied++ } } diff --git a/internal/store/certificatesigningrequest_test.go b/internal/store/certificatesigningrequest_test.go index 0244126460..990061942a 100644 --- a/internal/store/certificatesigningrequest_test.go +++ b/internal/store/certificatesigningrequest_test.go @@ -20,26 +20,26 @@ import ( "testing" "time" - certv1beta1 "k8s.io/api/certificates/v1beta1" + certv1 "k8s.io/api/certificates/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) func TestCsrStore(t *testing.T) { const metadata = ` - # HELP kube_certificatesigningrequest_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_certificatesigningrequest_labels [STABLE] Kubernetes labels converted to Prometheus labels. # TYPE kube_certificatesigningrequest_labels gauge - # HELP kube_certificatesigningrequest_created Unix creation timestamp + # HELP kube_certificatesigningrequest_created [STABLE] Unix creation timestamp # TYPE kube_certificatesigningrequest_created gauge - # HELP kube_certificatesigningrequest_condition The number of each certificatesigningrequest condition + # HELP kube_certificatesigningrequest_condition [STABLE] The number of each certificatesigningrequest condition # TYPE kube_certificatesigningrequest_condition gauge - # HELP kube_certificatesigningrequest_cert_length Length of the issued cert + # HELP kube_certificatesigningrequest_cert_length [STABLE] Length of the issued cert # TYPE kube_certificatesigningrequest_cert_length gauge ` cases := []generateMetricsTestCase{ { - Obj: &certv1beta1.CertificateSigningRequest{ + Obj: &certv1.CertificateSigningRequest{ ObjectMeta: metav1.ObjectMeta{ Name: "certificate-test", Generation: 1, @@ -48,20 +48,22 @@ func TestCsrStore(t *testing.T) { }, CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, }, - Status: certv1beta1.CertificateSigningRequestStatus{}, - Spec: certv1beta1.CertificateSigningRequestSpec{}, + Status: certv1.CertificateSigningRequestStatus{}, + Spec: certv1.CertificateSigningRequestSpec{ + SignerName: "signer", + }, }, Want: metadata + ` - kube_certificatesigningrequest_created{certificatesigningrequest="certificate-test"} 1.5e+09 - kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",condition="approved"} 0 - kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",condition="denied"} 0 - kube_certificatesigningrequest_labels{certificatesigningrequest="certificate-test",label_cert="test"} 1 - kube_certificatesigningrequest_cert_length{certificatesigningrequest="certificate-test"} 0 + kube_certificatesigningrequest_created{certificatesigningrequest="certificate-test",signer_name="signer"} 1.5e+09 + kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",signer_name="signer",condition="approved"} 0 + kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",signer_name="signer",condition="denied"} 0 + kube_certificatesigningrequest_labels{certificatesigningrequest="certificate-test",signer_name="signer"} 1 + kube_certificatesigningrequest_cert_length{certificatesigningrequest="certificate-test",signer_name="signer"} 0 `, MetricNames: []string{"kube_certificatesigningrequest_created", "kube_certificatesigningrequest_condition", "kube_certificatesigningrequest_labels", "kube_certificatesigningrequest_cert_length"}, }, { - Obj: &certv1beta1.CertificateSigningRequest{ + Obj: &certv1.CertificateSigningRequest{ ObjectMeta: metav1.ObjectMeta{ Name: "certificate-test", Generation: 1, @@ -70,26 +72,28 @@ func TestCsrStore(t *testing.T) { }, CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, }, - Status: certv1beta1.CertificateSigningRequestStatus{ - Conditions: []certv1beta1.CertificateSigningRequestCondition{ + Status: certv1.CertificateSigningRequestStatus{ + Conditions: []certv1.CertificateSigningRequestCondition{ { - Type: certv1beta1.CertificateDenied, + Type: certv1.CertificateDenied, }, }, }, - Spec: certv1beta1.CertificateSigningRequestSpec{}, + Spec: certv1.CertificateSigningRequestSpec{ + SignerName: "signer", + }, }, Want: metadata + ` - kube_certificatesigningrequest_created{certificatesigningrequest="certificate-test"} 1.5e+09 - kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",condition="approved"} 0 - kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",condition="denied"} 1 - kube_certificatesigningrequest_labels{certificatesigningrequest="certificate-test",label_cert="test"} 1 - kube_certificatesigningrequest_cert_length{certificatesigningrequest="certificate-test"} 0 + kube_certificatesigningrequest_created{certificatesigningrequest="certificate-test",signer_name="signer"} 1.5e+09 + kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",signer_name="signer",condition="approved"} 0 + kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",signer_name="signer",condition="denied"} 1 + kube_certificatesigningrequest_labels{certificatesigningrequest="certificate-test",signer_name="signer"} 1 + kube_certificatesigningrequest_cert_length{certificatesigningrequest="certificate-test",signer_name="signer"} 0 `, MetricNames: []string{"kube_certificatesigningrequest_created", "kube_certificatesigningrequest_condition", "kube_certificatesigningrequest_labels", "kube_certificatesigningrequest_cert_length"}, }, { - Obj: &certv1beta1.CertificateSigningRequest{ + Obj: &certv1.CertificateSigningRequest{ ObjectMeta: metav1.ObjectMeta{ Name: "certificate-test", Generation: 1, @@ -98,26 +102,28 @@ func TestCsrStore(t *testing.T) { }, CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, }, - Status: certv1beta1.CertificateSigningRequestStatus{ - Conditions: []certv1beta1.CertificateSigningRequestCondition{ + Status: certv1.CertificateSigningRequestStatus{ + Conditions: []certv1.CertificateSigningRequestCondition{ { - Type: certv1beta1.CertificateApproved, + Type: certv1.CertificateApproved, }, }, }, - Spec: certv1beta1.CertificateSigningRequestSpec{}, + Spec: certv1.CertificateSigningRequestSpec{ + SignerName: "signer", + }, }, Want: metadata + ` - kube_certificatesigningrequest_created{certificatesigningrequest="certificate-test"} 1.5e+09 - kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",condition="approved"} 1 - kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",condition="denied"} 0 - kube_certificatesigningrequest_labels{certificatesigningrequest="certificate-test",label_cert="test"} 1 - kube_certificatesigningrequest_cert_length{certificatesigningrequest="certificate-test"} 0 + kube_certificatesigningrequest_created{certificatesigningrequest="certificate-test",signer_name="signer"} 1.5e+09 + kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",signer_name="signer",condition="approved"} 1 + kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",signer_name="signer",condition="denied"} 0 + kube_certificatesigningrequest_labels{certificatesigningrequest="certificate-test",signer_name="signer"} 1 + kube_certificatesigningrequest_cert_length{certificatesigningrequest="certificate-test",signer_name="signer"} 0 `, MetricNames: []string{"kube_certificatesigningrequest_created", "kube_certificatesigningrequest_condition", "kube_certificatesigningrequest_labels", "kube_certificatesigningrequest_cert_length"}, }, { - Obj: &certv1beta1.CertificateSigningRequest{ + Obj: &certv1.CertificateSigningRequest{ ObjectMeta: metav1.ObjectMeta{ Name: "certificate-test", Generation: 1, @@ -126,26 +132,29 @@ func TestCsrStore(t *testing.T) { }, CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, }, - Status: certv1beta1.CertificateSigningRequestStatus{ + Spec: certv1.CertificateSigningRequestSpec{ + SignerName: "signer", + }, + Status: certv1.CertificateSigningRequestStatus{ Certificate: []byte("just for test"), - Conditions: []certv1beta1.CertificateSigningRequestCondition{ + Conditions: []certv1.CertificateSigningRequestCondition{ { - Type: certv1beta1.CertificateApproved, + Type: certv1.CertificateApproved, }, }, }, }, Want: metadata + ` - kube_certificatesigningrequest_created{certificatesigningrequest="certificate-test"} 1.5e+09 - kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",condition="approved"} 1 - kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",condition="denied"} 0 - kube_certificatesigningrequest_labels{certificatesigningrequest="certificate-test",label_cert="test"} 1 - kube_certificatesigningrequest_cert_length{certificatesigningrequest="certificate-test"} 13 + kube_certificatesigningrequest_created{certificatesigningrequest="certificate-test",signer_name="signer"} 1.5e+09 + kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",signer_name="signer",condition="approved"} 1 + kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",signer_name="signer",condition="denied"} 0 + kube_certificatesigningrequest_labels{certificatesigningrequest="certificate-test",signer_name="signer"} 1 + kube_certificatesigningrequest_cert_length{certificatesigningrequest="certificate-test",signer_name="signer"} 13 `, MetricNames: []string{"kube_certificatesigningrequest_created", "kube_certificatesigningrequest_condition", "kube_certificatesigningrequest_labels", "kube_certificatesigningrequest_cert_length"}, }, { - Obj: &certv1beta1.CertificateSigningRequest{ + Obj: &certv1.CertificateSigningRequest{ ObjectMeta: metav1.ObjectMeta{ Name: "certificate-test", Generation: 1, @@ -154,28 +163,31 @@ func TestCsrStore(t *testing.T) { }, CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, }, - Status: certv1beta1.CertificateSigningRequestStatus{ - Conditions: []certv1beta1.CertificateSigningRequestCondition{ + Spec: certv1.CertificateSigningRequestSpec{ + SignerName: "signer", + }, + Status: certv1.CertificateSigningRequestStatus{ + Conditions: []certv1.CertificateSigningRequestCondition{ { - Type: certv1beta1.CertificateApproved, + Type: certv1.CertificateApproved, }, { - Type: certv1beta1.CertificateDenied, + Type: certv1.CertificateDenied, }, }, }, }, Want: metadata + ` - kube_certificatesigningrequest_created{certificatesigningrequest="certificate-test"} 1.5e+09 - kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",condition="approved"} 1 - kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",condition="denied"} 1 - kube_certificatesigningrequest_labels{certificatesigningrequest="certificate-test",label_cert="test"} 1 - kube_certificatesigningrequest_cert_length{certificatesigningrequest="certificate-test"} 0 + kube_certificatesigningrequest_created{certificatesigningrequest="certificate-test",signer_name="signer"} 1.5e+09 + kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",signer_name="signer",condition="approved"} 1 + kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",signer_name="signer",condition="denied"} 1 + kube_certificatesigningrequest_labels{certificatesigningrequest="certificate-test",signer_name="signer"} 1 + kube_certificatesigningrequest_cert_length{certificatesigningrequest="certificate-test",signer_name="signer"} 0 `, MetricNames: []string{"kube_certificatesigningrequest_created", "kube_certificatesigningrequest_condition", "kube_certificatesigningrequest_labels", "kube_certificatesigningrequest_cert_length"}, }, { - Obj: &certv1beta1.CertificateSigningRequest{ + Obj: &certv1.CertificateSigningRequest{ ObjectMeta: metav1.ObjectMeta{ Name: "certificate-test", Generation: 1, @@ -184,36 +196,39 @@ func TestCsrStore(t *testing.T) { }, CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, }, - Status: certv1beta1.CertificateSigningRequestStatus{ - Conditions: []certv1beta1.CertificateSigningRequestCondition{ + Spec: certv1.CertificateSigningRequestSpec{ + SignerName: "signer", + }, + Status: certv1.CertificateSigningRequestStatus{ + Conditions: []certv1.CertificateSigningRequestCondition{ { - Type: certv1beta1.CertificateApproved, + Type: certv1.CertificateApproved, }, { - Type: certv1beta1.CertificateDenied, + Type: certv1.CertificateDenied, }, { - Type: certv1beta1.CertificateApproved, + Type: certv1.CertificateApproved, }, { - Type: certv1beta1.CertificateDenied, + Type: certv1.CertificateDenied, }, }, }, }, Want: metadata + ` - kube_certificatesigningrequest_created{certificatesigningrequest="certificate-test"} 1.5e+09 - kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",condition="approved"} 2 - kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",condition="denied"} 2 - kube_certificatesigningrequest_labels{certificatesigningrequest="certificate-test",label_cert="test"} 1 - kube_certificatesigningrequest_cert_length{certificatesigningrequest="certificate-test"} 0 + kube_certificatesigningrequest_created{certificatesigningrequest="certificate-test",signer_name="signer"} 1.5e+09 + kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",signer_name="signer",condition="approved"} 2 + kube_certificatesigningrequest_condition{certificatesigningrequest="certificate-test",signer_name="signer",condition="denied"} 2 + kube_certificatesigningrequest_labels{certificatesigningrequest="certificate-test",signer_name="signer"} 1 + kube_certificatesigningrequest_cert_length{certificatesigningrequest="certificate-test",signer_name="signer"} 0 `, MetricNames: []string{"kube_certificatesigningrequest_created", "kube_certificatesigningrequest_condition", "kube_certificatesigningrequest_labels", "kube_certificatesigningrequest_cert_length"}, }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(csrMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(csrMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(csrMetricFamilies(nil, nil)) + c.Headers = generator.ExtractMetricFamilyHeaders(csrMetricFamilies(nil, nil)) if err := c.run(); err != nil { t.Errorf("unexpected error when collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/clusterrole.go b/internal/store/clusterrole.go new file mode 100644 index 0000000000..d1d11a2875 --- /dev/null +++ b/internal/store/clusterrole.go @@ -0,0 +1,160 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "context" + + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +var ( + descClusterRoleAnnotationsName = "kube_clusterrole_annotations" + descClusterRoleAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." + descClusterRoleLabelsName = "kube_clusterrole_labels" + descClusterRoleLabelsHelp = "Kubernetes labels converted to Prometheus labels." + descClusterRoleLabelsDefaultLabels = []string{"clusterrole"} +) + +func clusterRoleMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + descClusterRoleAnnotationsName, + descClusterRoleAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapClusterRoleFunc(func(r *rbacv1.ClusterRole) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", r.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descClusterRoleLabelsName, + descClusterRoleLabelsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapClusterRoleFunc(func(r *rbacv1.ClusterRole) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", r.Labels, allowLabelsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: labelKeys, + LabelValues: labelValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_clusterrole_info", + "Information about cluster role.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapClusterRoleFunc(func(r *rbacv1.ClusterRole) *metric.Family { + return &metric.Family{ + Metrics: []*metric.Metric{{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: 1, + }}, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_clusterrole_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapClusterRoleFunc(func(r *rbacv1.ClusterRole) *metric.Family { + ms := []*metric.Metric{} + + if !r.CreationTimestamp.IsZero() { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64(r.CreationTimestamp.Unix()), + }) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_clusterrole_metadata_resource_version", + "Resource version representing a specific version of the cluster role.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapClusterRoleFunc(func(r *rbacv1.ClusterRole) *metric.Family { + return &metric.Family{ + Metrics: resourceVersionMetric(r.ObjectMeta.ResourceVersion), + } + }), + ), + } +} + +func createClusterRoleListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { + return &cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + opts.FieldSelector = fieldSelector + return kubeClient.RbacV1().ClusterRoles().List(context.TODO(), opts) + }, + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + opts.FieldSelector = fieldSelector + return kubeClient.RbacV1().ClusterRoles().Watch(context.TODO(), opts) + }, + } +} + +func wrapClusterRoleFunc(f func(*rbacv1.ClusterRole) *metric.Family) func(interface{}) *metric.Family { + return func(obj interface{}) *metric.Family { + clusterrole := obj.(*rbacv1.ClusterRole) + + metricFamily := f(clusterrole) + + for _, m := range metricFamily.Metrics { + m.LabelKeys, m.LabelValues = mergeKeyValues(descClusterRoleLabelsDefaultLabels, []string{clusterrole.Name}, m.LabelKeys, m.LabelValues) + } + + return metricFamily + } +} diff --git a/internal/store/clusterrole_test.go b/internal/store/clusterrole_test.go new file mode 100644 index 0000000000..aef2d2659d --- /dev/null +++ b/internal/store/clusterrole_test.go @@ -0,0 +1,103 @@ +/* +Copyright 2012 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "testing" + + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +func TestClusterRoleStore(t *testing.T) { + startTime := 1501569018 + metav1StartTime := metav1.Unix(int64(startTime), 0) + + cases := []generateMetricsTestCase{ + { + AllowAnnotationsList: []string{ + "app.k8s.io/owner", + }, + AllowLabelsList: []string{ + "app", + }, + Obj: &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "role1", + ResourceVersion: "BBBBB", + Annotations: map[string]string{ + "app": "mysql-server", + "app.k8s.io/owner": "@foo", + }, + Labels: map[string]string{ + "excluded": "me", + "app": "mysql-server", + }, + }, + }, + Want: ` + # HELP kube_clusterrole_annotations Kubernetes annotations converted to Prometheus labels. + # HELP kube_clusterrole_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_clusterrole_info Information about cluster role. + # HELP kube_clusterrole_metadata_resource_version Resource version representing a specific version of the cluster role. + # TYPE kube_clusterrole_annotations gauge + # TYPE kube_clusterrole_labels gauge + # TYPE kube_clusterrole_info gauge + # TYPE kube_clusterrole_metadata_resource_version gauge + kube_clusterrole_annotations{annotation_app_k8s_io_owner="@foo",clusterrole="role1"} 1 + kube_clusterrole_labels{clusterrole="role1",label_app="mysql-server"} 1 + kube_clusterrole_info{clusterrole="role1"} 1 +`, + MetricNames: []string{ + "kube_clusterrole_annotations", + "kube_clusterrole_labels", + "kube_clusterrole_info", + "kube_clusterrole_metadata_resource_version", + }, + }, + { + Obj: &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "role2", + CreationTimestamp: metav1StartTime, + ResourceVersion: "10596", + }, + }, + Want: ` + # HELP kube_clusterrole_created Unix creation timestamp + # HELP kube_clusterrole_info Information about cluster role. + # HELP kube_clusterrole_metadata_resource_version Resource version representing a specific version of the cluster role. + # TYPE kube_clusterrole_created gauge + # TYPE kube_clusterrole_info gauge + # TYPE kube_clusterrole_metadata_resource_version gauge + kube_clusterrole_info{clusterrole="role2"} 1 + kube_clusterrole_created{clusterrole="role2"} 1.501569018e+09 + kube_clusterrole_metadata_resource_version{clusterrole="role2"} 10596 + `, + MetricNames: []string{"kube_clusterrole_info", "kube_clusterrole_created", "kube_clusterrole_metadata_resource_version"}, + }, + } + for i, c := range cases { + c.Func = generator.ComposeMetricGenFuncs(clusterRoleMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) + c.Headers = generator.ExtractMetricFamilyHeaders(clusterRoleMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) + if err := c.run(); err != nil { + t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) + } + } +} diff --git a/internal/store/clusterrolebinding.go b/internal/store/clusterrolebinding.go new file mode 100644 index 0000000000..2a88e612e4 --- /dev/null +++ b/internal/store/clusterrolebinding.go @@ -0,0 +1,162 @@ +/* +Copyright 2022 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "context" + + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +var ( + descClusterRoleBindingAnnotationsName = "kube_clusterrolebinding_annotations" + descClusterRoleBindingAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." + descClusterRoleBindingLabelsName = "kube_clusterrolebinding_labels" + descClusterRoleBindingLabelsHelp = "Kubernetes labels converted to Prometheus labels." + descClusterRoleBindingLabelsDefaultLabels = []string{"clusterrolebinding"} +) + +func clusterRoleBindingMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + descClusterRoleBindingAnnotationsName, + descClusterRoleBindingAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapClusterRoleBindingFunc(func(r *rbacv1.ClusterRoleBinding) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", r.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descClusterRoleBindingLabelsName, + descClusterRoleBindingLabelsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapClusterRoleBindingFunc(func(r *rbacv1.ClusterRoleBinding) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", r.Labels, allowLabelsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: labelKeys, + LabelValues: labelValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_clusterrolebinding_info", + "Information about clusterrolebinding.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapClusterRoleBindingFunc(func(r *rbacv1.ClusterRoleBinding) *metric.Family { + labelKeys := []string{"roleref_kind", "roleref_name"} + labelValues := []string{r.RoleRef.Kind, r.RoleRef.Name} + return &metric.Family{ + Metrics: []*metric.Metric{{ + LabelKeys: labelKeys, + LabelValues: labelValues, + Value: 1, + }}, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_clusterrolebinding_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapClusterRoleBindingFunc(func(r *rbacv1.ClusterRoleBinding) *metric.Family { + ms := []*metric.Metric{} + + if !r.CreationTimestamp.IsZero() { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64(r.CreationTimestamp.Unix()), + }) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_clusterrolebinding_metadata_resource_version", + "Resource version representing a specific version of the clusterrolebinding.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapClusterRoleBindingFunc(func(r *rbacv1.ClusterRoleBinding) *metric.Family { + return &metric.Family{ + Metrics: resourceVersionMetric(r.ObjectMeta.ResourceVersion), + } + }), + ), + } +} + +func createClusterRoleBindingListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { + return &cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + opts.FieldSelector = fieldSelector + return kubeClient.RbacV1().ClusterRoleBindings().List(context.TODO(), opts) + }, + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + opts.FieldSelector = fieldSelector + return kubeClient.RbacV1().ClusterRoleBindings().Watch(context.TODO(), opts) + }, + } +} + +func wrapClusterRoleBindingFunc(f func(*rbacv1.ClusterRoleBinding) *metric.Family) func(interface{}) *metric.Family { + return func(obj interface{}) *metric.Family { + clusterrolebinding := obj.(*rbacv1.ClusterRoleBinding) + + metricFamily := f(clusterrolebinding) + + for _, m := range metricFamily.Metrics { + m.LabelKeys, m.LabelValues = mergeKeyValues(descClusterRoleBindingLabelsDefaultLabels, []string{clusterrolebinding.Name}, m.LabelKeys, m.LabelValues) + } + + return metricFamily + } +} diff --git a/internal/store/clusterrolebinding_test.go b/internal/store/clusterrolebinding_test.go new file mode 100644 index 0000000000..e367425853 --- /dev/null +++ b/internal/store/clusterrolebinding_test.go @@ -0,0 +1,113 @@ +/* +Copyright 2022 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "testing" + + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +func TestClusterRoleBindingStore(t *testing.T) { + startTime := 1501569018 + metav1StartTime := metav1.Unix(int64(startTime), 0) + + cases := []generateMetricsTestCase{ + { + AllowAnnotationsList: []string{ + "app.k8s.io/owner", + }, + AllowLabelsList: []string{ + "app", + }, + Obj: &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "clusterrolebinding1", + ResourceVersion: "BBBBB", + Annotations: map[string]string{ + "app": "mysql-server", + "app.k8s.io/owner": "@foo", + }, + Labels: map[string]string{ + "excluded": "me", + "app": "mysql-server", + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "role", + }, + }, + Want: ` + # HELP kube_clusterrolebinding_annotations Kubernetes annotations converted to Prometheus labels. + # HELP kube_clusterrolebinding_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_clusterrolebinding_info Information about clusterrolebinding. + # HELP kube_clusterrolebinding_metadata_resource_version Resource version representing a specific version of the clusterrolebinding. + # TYPE kube_clusterrolebinding_annotations gauge + # TYPE kube_clusterrolebinding_labels gauge + # TYPE kube_clusterrolebinding_info gauge + # TYPE kube_clusterrolebinding_metadata_resource_version gauge + kube_clusterrolebinding_annotations{annotation_app_k8s_io_owner="@foo",clusterrolebinding="clusterrolebinding1"} 1 + kube_clusterrolebinding_labels{clusterrolebinding="clusterrolebinding1",label_app="mysql-server"} 1 + kube_clusterrolebinding_info{clusterrolebinding="clusterrolebinding1",roleref_kind="Role",roleref_name="role"} 1 +`, + MetricNames: []string{ + "kube_clusterrolebinding_annotations", + "kube_clusterrolebinding_labels", + "kube_clusterrolebinding_info", + "kube_clusterrolebinding_metadata_resource_version", + }, + }, + { + Obj: &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "clusterrolebinding2", + CreationTimestamp: metav1StartTime, + ResourceVersion: "10596", + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "role", + }, + }, + Want: ` + # HELP kube_clusterrolebinding_created Unix creation timestamp + # HELP kube_clusterrolebinding_info Information about clusterrolebinding. + # HELP kube_clusterrolebinding_metadata_resource_version Resource version representing a specific version of the clusterrolebinding. + # TYPE kube_clusterrolebinding_created gauge + # TYPE kube_clusterrolebinding_info gauge + # TYPE kube_clusterrolebinding_metadata_resource_version gauge + kube_clusterrolebinding_info{clusterrolebinding="clusterrolebinding2",roleref_kind="Role",roleref_name="role"} 1 + kube_clusterrolebinding_created{clusterrolebinding="clusterrolebinding2"} 1.501569018e+09 + kube_clusterrolebinding_metadata_resource_version{clusterrolebinding="clusterrolebinding2"} 10596 + `, + MetricNames: []string{"kube_clusterrolebinding_info", "kube_clusterrolebinding_created", "kube_clusterrolebinding_metadata_resource_version"}, + }, + } + for i, c := range cases { + c.Func = generator.ComposeMetricGenFuncs(clusterRoleBindingMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) + c.Headers = generator.ExtractMetricFamilyHeaders(clusterRoleBindingMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) + if err := c.run(); err != nil { + t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) + } + } +} diff --git a/internal/store/configmap.go b/internal/store/configmap.go index 2abfa216e3..619274d549 100644 --- a/internal/store/configmap.go +++ b/internal/store/configmap.go @@ -17,25 +17,71 @@ limitations under the License. package store import ( + "context" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" - "k8s.io/kube-state-metrics/pkg/metric" + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) var ( descConfigMapLabelsDefaultLabels = []string{"namespace", "configmap"} +) - configMapMetricFamilies = []metric.FamilyGenerator{ - { - Name: "kube_configmap_info", - Type: metric.Gauge, - Help: "Information about configmap.", - GenerateFunc: wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family { +func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_configmap_annotations", + "Kubernetes annotations converted to Prometheus labels.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", c.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_configmap_labels", + "Kubernetes labels converted to Prometheus labels.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", c.Labels, allowLabelsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: labelKeys, + LabelValues: labelValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_configmap_info", + "Information about configmap.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{{ LabelKeys: []string{}, @@ -44,12 +90,14 @@ var ( }}, } }), - }, - { - Name: "kube_configmap_created", - Type: metric.Gauge, - Help: "Unix creation timestamp", - GenerateFunc: wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_configmap_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.STABLE, + "", + wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family { ms := []*metric.Metric{} if !c.CreationTimestamp.IsZero() { @@ -64,27 +112,31 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_configmap_metadata_resource_version", - Type: metric.Gauge, - Help: "Resource version representing a specific version of the configmap.", - GenerateFunc: wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_configmap_metadata_resource_version", + "Resource version representing a specific version of the configmap.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family { return &metric.Family{ Metrics: resourceVersionMetric(c.ObjectMeta.ResourceVersion), } }), - }, + ), } -) +} -func createConfigMapListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createConfigMapListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.CoreV1().ConfigMaps(ns).List(opts) + opts.FieldSelector = fieldSelector + return kubeClient.CoreV1().ConfigMaps(ns).List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.CoreV1().ConfigMaps(ns).Watch(opts) + opts.FieldSelector = fieldSelector + return kubeClient.CoreV1().ConfigMaps(ns).Watch(context.TODO(), opts) }, } } @@ -96,8 +148,7 @@ func wrapConfigMapFunc(f func(*v1.ConfigMap) *metric.Family) func(interface{}) * metricFamily := f(configMap) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descConfigMapLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{configMap.Namespace, configMap.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descConfigMapLabelsDefaultLabels, []string{configMap.Namespace, configMap.Name}, m.LabelKeys, m.LabelValues) } return metricFamily diff --git a/internal/store/configmap_test.go b/internal/store/configmap_test.go index 40dc0dc7a4..aecf13c4d7 100644 --- a/internal/store/configmap_test.go +++ b/internal/store/configmap_test.go @@ -22,7 +22,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) func TestConfigMapStore(t *testing.T) { @@ -31,21 +31,46 @@ func TestConfigMapStore(t *testing.T) { cases := []generateMetricsTestCase{ { + AllowAnnotationsList: []string{ + "app.k8s.io/owner", + }, + AllowLabelsList: []string{ + "app", + }, Obj: &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "configmap1", Namespace: "ns1", ResourceVersion: "BBBBB", + Annotations: map[string]string{ + "app": "mysql-server", + "app.k8s.io/owner": "@foo", + }, + Labels: map[string]string{ + "excluded": "me", + "app": "mysql-server", + }, }, }, Want: ` - # HELP kube_configmap_info Information about configmap. + # HELP kube_configmap_annotations Kubernetes annotations converted to Prometheus labels. + # HELP kube_configmap_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # HELP kube_configmap_info [STABLE] Information about configmap. # HELP kube_configmap_metadata_resource_version Resource version representing a specific version of the configmap. + # TYPE kube_configmap_annotations gauge + # TYPE kube_configmap_labels gauge # TYPE kube_configmap_info gauge # TYPE kube_configmap_metadata_resource_version gauge + kube_configmap_annotations{annotation_app_k8s_io_owner="@foo",configmap="configmap1",namespace="ns1"} 1 + kube_configmap_labels{configmap="configmap1",label_app="mysql-server",namespace="ns1"} 1 kube_configmap_info{configmap="configmap1",namespace="ns1"} 1 `, - MetricNames: []string{"kube_configmap_info", "kube_configmap_metadata_resource_version"}, + MetricNames: []string{ + "kube_configmap_annotations", + "kube_configmap_labels", + "kube_configmap_info", + "kube_configmap_metadata_resource_version", + }, }, { Obj: &v1.ConfigMap{ @@ -57,8 +82,8 @@ func TestConfigMapStore(t *testing.T) { }, }, Want: ` - # HELP kube_configmap_created Unix creation timestamp - # HELP kube_configmap_info Information about configmap. + # HELP kube_configmap_created [STABLE] Unix creation timestamp + # HELP kube_configmap_info [STABLE] Information about configmap. # HELP kube_configmap_metadata_resource_version Resource version representing a specific version of the configmap. # TYPE kube_configmap_created gauge # TYPE kube_configmap_info gauge @@ -71,8 +96,8 @@ func TestConfigMapStore(t *testing.T) { }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(configMapMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(configMapMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(configMapMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) + c.Headers = generator.ExtractMetricFamilyHeaders(configMapMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/cronjob.go b/internal/store/cronjob.go index af4eed526c..5976b6c03d 100644 --- a/internal/store/cronjob.go +++ b/internal/store/cronjob.go @@ -17,32 +17,61 @@ limitations under the License. package store import ( + "context" + "errors" + "fmt" "time" - "github.com/pkg/errors" "github.com/robfig/cron/v3" - batchv1beta1 "k8s.io/api/batch/v1beta1" + batchv1 "k8s.io/api/batch/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" - "k8s.io/kube-state-metrics/pkg/metric" + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) var ( + descCronJobAnnotationsName = "kube_cronjob_annotations" + descCronJobAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." descCronJobLabelsName = "kube_cronjob_labels" descCronJobLabelsHelp = "Kubernetes labels converted to Prometheus labels." descCronJobLabelsDefaultLabels = []string{"namespace", "cronjob"} +) - cronJobMetricFamilies = []metric.FamilyGenerator{ - { - Name: descCronJobLabelsName, - Type: metric.Gauge, - Help: descCronJobLabelsHelp, - GenerateFunc: wrapCronJobFunc(func(j *batchv1beta1.CronJob) *metric.Family { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(j.Labels) +func cronJobMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + descCronJobAnnotationsName, + descCronJobAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapCronJobFunc(func(j *batchv1.CronJob) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", j.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descCronJobLabelsName, + descCronJobLabelsHelp, + metric.Gauge, + basemetrics.STABLE, + "", + wrapCronJobFunc(func(j *batchv1.CronJob) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", j.Labels, allowLabelsList) return &metric.Family{ Metrics: []*metric.Metric{ { @@ -53,12 +82,14 @@ var ( }, } }), - }, - { - Name: "kube_cronjob_info", - Type: metric.Gauge, - Help: "Info about cronjob.", - GenerateFunc: wrapCronJobFunc(func(j *batchv1beta1.CronJob) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_cronjob_info", + "Info about cronjob.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapCronJobFunc(func(j *batchv1.CronJob) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -69,12 +100,14 @@ var ( }, } }), - }, - { - Name: "kube_cronjob_created", - Type: metric.Gauge, - Help: "Unix creation timestamp", - GenerateFunc: wrapCronJobFunc(func(j *batchv1beta1.CronJob) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_cronjob_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.STABLE, + "", + wrapCronJobFunc(func(j *batchv1.CronJob) *metric.Family { ms := []*metric.Metric{} if !j.CreationTimestamp.IsZero() { ms = append(ms, &metric.Metric{ @@ -88,12 +121,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_cronjob_status_active", - Type: metric.Gauge, - Help: "Active holds pointers to currently running jobs.", - GenerateFunc: wrapCronJobFunc(func(j *batchv1beta1.CronJob) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_cronjob_status_active", + "Active holds pointers to currently running jobs.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapCronJobFunc(func(j *batchv1.CronJob) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -104,12 +139,14 @@ var ( }, } }), - }, - { - Name: "kube_cronjob_status_last_schedule_time", - Type: metric.Gauge, - Help: "LastScheduleTime keeps information of when was the last time the job was successfully scheduled.", - GenerateFunc: wrapCronJobFunc(func(j *batchv1beta1.CronJob) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_cronjob_status_last_schedule_time", + "LastScheduleTime keeps information of when was the last time the job was successfully scheduled.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapCronJobFunc(func(j *batchv1.CronJob) *metric.Family { ms := []*metric.Metric{} if j.Status.LastScheduleTime != nil { @@ -124,12 +161,36 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_cronjob_spec_suspend", - Type: metric.Gauge, - Help: "Suspend flag tells the controller to suspend subsequent executions.", - GenerateFunc: wrapCronJobFunc(func(j *batchv1beta1.CronJob) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_cronjob_status_last_successful_time", + "LastSuccessfulTime keeps information of when was the last time the job was completed successfully.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapCronJobFunc(func(j *batchv1.CronJob) *metric.Family { + ms := []*metric.Metric{} + + if j.Status.LastSuccessfulTime != nil { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64(j.Status.LastSuccessfulTime.Unix()), + }) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_cronjob_spec_suspend", + "Suspend flag tells the controller to suspend subsequent executions.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapCronJobFunc(func(j *batchv1.CronJob) *metric.Family { ms := []*metric.Metric{} if j.Spec.Suspend != nil { @@ -144,12 +205,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_cronjob_spec_starting_deadline_seconds", - Type: metric.Gauge, - Help: "Deadline in seconds for starting the job if it misses scheduled time for any reason.", - GenerateFunc: wrapCronJobFunc(func(j *batchv1beta1.CronJob) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_cronjob_spec_starting_deadline_seconds", + "Deadline in seconds for starting the job if it misses scheduled time for any reason.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapCronJobFunc(func(j *batchv1.CronJob) *metric.Family { ms := []*metric.Metric{} if j.Spec.StartingDeadlineSeconds != nil { @@ -165,12 +228,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_cronjob_next_schedule_time", - Type: metric.Gauge, - Help: "Next time the cronjob should be scheduled. The time after lastScheduleTime, or after the cron job's creation time if it's never been scheduled. Use this to determine if the job is delayed.", - GenerateFunc: wrapCronJobFunc(func(j *batchv1beta1.CronJob) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_cronjob_next_schedule_time", + "Next time the cronjob should be scheduled. The time after lastScheduleTime, or after the cron job's creation time if it's never been scheduled. Use this to determine if the job is delayed.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapCronJobFunc(func(j *batchv1.CronJob) *metric.Family { ms := []*metric.Metric{} // If the cron job is suspended, don't track the next scheduled time @@ -189,32 +254,89 @@ var ( Metrics: ms, } }), - }, + ), + *generator.NewFamilyGeneratorWithStability( + "kube_cronjob_metadata_resource_version", + "Resource version representing a specific version of the cronjob.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapCronJobFunc(func(j *batchv1.CronJob) *metric.Family { + return &metric.Family{ + Metrics: resourceVersionMetric(j.ObjectMeta.ResourceVersion), + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_cronjob_spec_successful_job_history_limit", + "Successful job history limit tells the controller how many completed jobs should be preserved.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapCronJobFunc(func(j *batchv1.CronJob) *metric.Family { + ms := []*metric.Metric{} + + if j.Spec.SuccessfulJobsHistoryLimit != nil { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64(*j.Spec.SuccessfulJobsHistoryLimit), + }) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_cronjob_spec_failed_job_history_limit", + "Failed job history limit tells the controller how many failed jobs should be preserved.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapCronJobFunc(func(j *batchv1.CronJob) *metric.Family { + ms := []*metric.Metric{} + + if j.Spec.FailedJobsHistoryLimit != nil { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64(*j.Spec.FailedJobsHistoryLimit), + }) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ), } -) +} -func wrapCronJobFunc(f func(*batchv1beta1.CronJob) *metric.Family) func(interface{}) *metric.Family { +func wrapCronJobFunc(f func(*batchv1.CronJob) *metric.Family) func(interface{}) *metric.Family { return func(obj interface{}) *metric.Family { - cronJob := obj.(*batchv1beta1.CronJob) + cronJob := obj.(*batchv1.CronJob) metricFamily := f(cronJob) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descCronJobLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{cronJob.Namespace, cronJob.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descCronJobLabelsDefaultLabels, []string{cronJob.Namespace, cronJob.Name}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createCronJobListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createCronJobListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.BatchV1beta1().CronJobs(ns).List(opts) + opts.FieldSelector = fieldSelector + return kubeClient.BatchV1().CronJobs(ns).List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.BatchV1beta1().CronJobs(ns).Watch(opts) + opts.FieldSelector = fieldSelector + return kubeClient.BatchV1().CronJobs(ns).Watch(context.TODO(), opts) }, } } @@ -222,7 +344,7 @@ func createCronJobListWatch(kubeClient clientset.Interface, ns string) cache.Lis func getNextScheduledTime(schedule string, lastScheduleTime *metav1.Time, createdTime metav1.Time) (time.Time, error) { sched, err := cron.ParseStandard(schedule) if err != nil { - return time.Time{}, errors.Wrapf(err, "Failed to parse cron job schedule '%s'", schedule) + return time.Time{}, fmt.Errorf("Failed to parse cron job schedule '%s': %w", schedule, err) } if !lastScheduleTime.IsZero() { return sched.Next(lastScheduleTime.Time), nil diff --git a/internal/store/cronjob_test.go b/internal/store/cronjob_test.go index 5c972dda73..9b992874b2 100644 --- a/internal/store/cronjob_test.go +++ b/internal/store/cronjob_test.go @@ -22,17 +22,19 @@ import ( "testing" "time" - batchv1beta1 "k8s.io/api/batch/v1beta1" + batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) var ( SuspendTrue = true SuspendFalse = false StartingDeadlineSeconds300 int64 = 300 + SuccessfulJobHistoryLimit3 int32 = 3 + FailedJobHistoryLimit1 int32 = 1 // "1520742896" is "2018/3/11 12:34:56" in "Asia/Shanghai". ActiveRunningCronJob1LastScheduleTime = time.Unix(1520742896, 0) @@ -101,150 +103,277 @@ func TestCronJobStore(t *testing.T) { cases := []generateMetricsTestCase{ { - Obj: &batchv1beta1.CronJob{ + AllowAnnotationsList: []string{ + "app.k8s.io/owner", + }, + Obj: &batchv1.CronJob{ ObjectMeta: metav1.ObjectMeta{ - Name: "ActiveRunningCronJob1", - Namespace: "ns1", - Generation: 1, + Name: "ActiveRunningCronJob1", + Namespace: "ns1", + Generation: 1, + ResourceVersion: "11111", Labels: map[string]string{ "app": "example-active-running-1", }, + Annotations: map[string]string{ + "app": "mysql-server", + "app.k8s.io/owner": "@foo", + }, }, - Status: batchv1beta1.CronJobStatus{ - Active: []v1.ObjectReference{{Name: "FakeJob1"}, {Name: "FakeJob2"}}, - LastScheduleTime: &metav1.Time{Time: ActiveRunningCronJob1LastScheduleTime}, + Status: batchv1.CronJobStatus{ + Active: []v1.ObjectReference{{Name: "FakeJob1"}, {Name: "FakeJob2"}}, + LastScheduleTime: &metav1.Time{Time: ActiveRunningCronJob1LastScheduleTime}, + LastSuccessfulTime: nil, }, - Spec: batchv1beta1.CronJobSpec{ - StartingDeadlineSeconds: &StartingDeadlineSeconds300, - ConcurrencyPolicy: "Forbid", - Suspend: &SuspendFalse, - Schedule: "0 */6 * * *", + Spec: batchv1.CronJobSpec{ + StartingDeadlineSeconds: &StartingDeadlineSeconds300, + ConcurrencyPolicy: "Forbid", + Suspend: &SuspendFalse, + Schedule: "0 */6 * * *", + SuccessfulJobsHistoryLimit: &SuccessfulJobHistoryLimit3, + FailedJobsHistoryLimit: &FailedJobHistoryLimit1, }, }, Want: ` - # HELP kube_cronjob_created Unix creation timestamp - # HELP kube_cronjob_info Info about cronjob. - # HELP kube_cronjob_labels Kubernetes labels converted to Prometheus labels. - # HELP kube_cronjob_next_schedule_time Next time the cronjob should be scheduled. The time after lastScheduleTime, or after the cron job's creation time if it's never been scheduled. Use this to determine if the job is delayed. - # HELP kube_cronjob_spec_starting_deadline_seconds Deadline in seconds for starting the job if it misses scheduled time for any reason. - # HELP kube_cronjob_spec_suspend Suspend flag tells the controller to suspend subsequent executions. - # HELP kube_cronjob_status_active Active holds pointers to currently running jobs. - # HELP kube_cronjob_status_last_schedule_time LastScheduleTime keeps information of when was the last time the job was successfully scheduled. + # HELP kube_cronjob_created [STABLE] Unix creation timestamp + # HELP kube_cronjob_info [STABLE] Info about cronjob. + # HELP kube_cronjob_annotations Kubernetes annotations converted to Prometheus labels. + # HELP kube_cronjob_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # HELP kube_cronjob_next_schedule_time [STABLE] Next time the cronjob should be scheduled. The time after lastScheduleTime, or after the cron job's creation time if it's never been scheduled. Use this to determine if the job is delayed. + # HELP kube_cronjob_spec_failed_job_history_limit Failed job history limit tells the controller how many failed jobs should be preserved. + # HELP kube_cronjob_spec_starting_deadline_seconds [STABLE] Deadline in seconds for starting the job if it misses scheduled time for any reason. + # HELP kube_cronjob_spec_successful_job_history_limit Successful job history limit tells the controller how many completed jobs should be preserved. + # HELP kube_cronjob_spec_suspend [STABLE] Suspend flag tells the controller to suspend subsequent executions. + # HELP kube_cronjob_status_active [STABLE] Active holds pointers to currently running jobs. + # HELP kube_cronjob_metadata_resource_version [STABLE] Resource version representing a specific version of the cronjob. + # HELP kube_cronjob_status_last_schedule_time [STABLE] LastScheduleTime keeps information of when was the last time the job was successfully scheduled. # TYPE kube_cronjob_created gauge # TYPE kube_cronjob_info gauge + # TYPE kube_cronjob_annotations gauge # TYPE kube_cronjob_labels gauge # TYPE kube_cronjob_next_schedule_time gauge + # TYPE kube_cronjob_spec_failed_job_history_limit gauge # TYPE kube_cronjob_spec_starting_deadline_seconds gauge + # TYPE kube_cronjob_spec_successful_job_history_limit gauge # TYPE kube_cronjob_spec_suspend gauge # TYPE kube_cronjob_status_active gauge + # TYPE kube_cronjob_metadata_resource_version gauge # TYPE kube_cronjob_status_last_schedule_time gauge kube_cronjob_info{concurrency_policy="Forbid",cronjob="ActiveRunningCronJob1",namespace="ns1",schedule="0 */6 * * *"} 1 - kube_cronjob_labels{cronjob="ActiveRunningCronJob1",label_app="example-active-running-1",namespace="ns1"} 1 + kube_cronjob_annotations{annotation_app_k8s_io_owner="@foo",cronjob="ActiveRunningCronJob1",namespace="ns1"} 1 + kube_cronjob_labels{cronjob="ActiveRunningCronJob1",namespace="ns1"} 1 + kube_cronjob_spec_failed_job_history_limit{cronjob="ActiveRunningCronJob1",namespace="ns1"} 1 kube_cronjob_spec_starting_deadline_seconds{cronjob="ActiveRunningCronJob1",namespace="ns1"} 300 + kube_cronjob_spec_successful_job_history_limit{cronjob="ActiveRunningCronJob1",namespace="ns1"} 3 kube_cronjob_spec_suspend{cronjob="ActiveRunningCronJob1",namespace="ns1"} 0 kube_cronjob_status_active{cronjob="ActiveRunningCronJob1",namespace="ns1"} 2 + kube_cronjob_metadata_resource_version{cronjob="ActiveRunningCronJob1",namespace="ns1"} 11111 kube_cronjob_status_last_schedule_time{cronjob="ActiveRunningCronJob1",namespace="ns1"} 1.520742896e+09 ` + fmt.Sprintf("kube_cronjob_next_schedule_time{cronjob=\"ActiveRunningCronJob1\",namespace=\"ns1\"} %ve+09\n", float64(ActiveRunningCronJob1NextScheduleTime.Unix())/math.Pow10(9)), - MetricNames: []string{"kube_cronjob_next_schedule_time", "kube_cronjob_spec_starting_deadline_seconds", "kube_cronjob_status_active", "kube_cronjob_spec_suspend", "kube_cronjob_info", "kube_cronjob_created", "kube_cronjob_labels", "kube_cronjob_status_last_schedule_time"}, + MetricNames: []string{ + "kube_cronjob_next_schedule_time", + "kube_cronjob_spec_starting_deadline_seconds", + "kube_cronjob_status_active", + "kube_cronjob_metadata_resource_version", + "kube_cronjob_spec_suspend", + "kube_cronjob_info", + "kube_cronjob_created", + "kube_cronjob_annotations", + "kube_cronjob_labels", + "kube_cronjob_status_last_schedule_time", + "kube_cronjob_spec_successful_job_history_limit", + "kube_cronjob_spec_failed_job_history_limit", + }, + }, + { + Obj: &batchv1.CronJob{ + ObjectMeta: metav1.ObjectMeta{ + Name: "SuspendedCronJob1", + Namespace: "ns1", + Generation: 1, + ResourceVersion: "22222", + Labels: map[string]string{ + "app": "example-suspended-1", + }, + }, + Status: batchv1.CronJobStatus{ + Active: []v1.ObjectReference{}, + LastScheduleTime: &metav1.Time{Time: SuspendedCronJob1LastScheduleTime}, + LastSuccessfulTime: nil, + }, + Spec: batchv1.CronJobSpec{ + StartingDeadlineSeconds: &StartingDeadlineSeconds300, + ConcurrencyPolicy: "Forbid", + Suspend: &SuspendTrue, + Schedule: "0 */3 * * *", + SuccessfulJobsHistoryLimit: &SuccessfulJobHistoryLimit3, + FailedJobsHistoryLimit: &FailedJobHistoryLimit1, + }, + }, + Want: ` + # HELP kube_cronjob_created [STABLE] Unix creation timestamp + # HELP kube_cronjob_info [STABLE] Info about cronjob. + # HELP kube_cronjob_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # HELP kube_cronjob_spec_failed_job_history_limit Failed job history limit tells the controller how many failed jobs should be preserved. + # HELP kube_cronjob_spec_starting_deadline_seconds [STABLE] Deadline in seconds for starting the job if it misses scheduled time for any reason. + # HELP kube_cronjob_spec_successful_job_history_limit Successful job history limit tells the controller how many completed jobs should be preserved. + # HELP kube_cronjob_spec_suspend [STABLE] Suspend flag tells the controller to suspend subsequent executions. + # HELP kube_cronjob_status_active [STABLE] Active holds pointers to currently running jobs. + # HELP kube_cronjob_metadata_resource_version [STABLE] Resource version representing a specific version of the cronjob. + # HELP kube_cronjob_status_last_schedule_time [STABLE] LastScheduleTime keeps information of when was the last time the job was successfully scheduled. + # HELP kube_cronjob_status_last_successful_time LastSuccessfulTime keeps information of when was the last time the job was completed successfully. + # TYPE kube_cronjob_created gauge + # TYPE kube_cronjob_info gauge + # TYPE kube_cronjob_labels gauge + # TYPE kube_cronjob_spec_failed_job_history_limit gauge + # TYPE kube_cronjob_spec_starting_deadline_seconds gauge + # TYPE kube_cronjob_spec_successful_job_history_limit gauge + # TYPE kube_cronjob_spec_suspend gauge + # TYPE kube_cronjob_status_active gauge + # TYPE kube_cronjob_metadata_resource_version gauge + # TYPE kube_cronjob_status_last_schedule_time gauge + # TYPE kube_cronjob_status_last_successful_time gauge + kube_cronjob_info{concurrency_policy="Forbid",cronjob="SuspendedCronJob1",namespace="ns1",schedule="0 */3 * * *"} 1 + kube_cronjob_labels{cronjob="SuspendedCronJob1",namespace="ns1"} 1 + kube_cronjob_spec_failed_job_history_limit{cronjob="SuspendedCronJob1",namespace="ns1"} 1 + kube_cronjob_spec_starting_deadline_seconds{cronjob="SuspendedCronJob1",namespace="ns1"} 300 + kube_cronjob_spec_successful_job_history_limit{cronjob="SuspendedCronJob1",namespace="ns1"} 3 + kube_cronjob_spec_suspend{cronjob="SuspendedCronJob1",namespace="ns1"} 1 + kube_cronjob_status_active{cronjob="SuspendedCronJob1",namespace="ns1"} 0 + kube_cronjob_metadata_resource_version{cronjob="SuspendedCronJob1",namespace="ns1"} 22222 + kube_cronjob_status_last_schedule_time{cronjob="SuspendedCronJob1",namespace="ns1"} 1.520762696e+09 +`, + MetricNames: []string{"kube_cronjob_status_last_successful_time", "kube_cronjob_spec_starting_deadline_seconds", "kube_cronjob_status_active", "kube_cronjob_metadata_resource_version", "kube_cronjob_spec_suspend", "kube_cronjob_info", "kube_cronjob_created", "kube_cronjob_labels", "kube_cronjob_status_last_schedule_time", "kube_cronjob_spec_successful_job_history_limit", "kube_cronjob_spec_failed_job_history_limit"}, }, { - Obj: &batchv1beta1.CronJob{ + Obj: &batchv1.CronJob{ ObjectMeta: metav1.ObjectMeta{ - Name: "SuspendedCronJob1", - Namespace: "ns1", - Generation: 1, + Name: "SuspendedCronJob1", + Namespace: "ns1", + Generation: 1, + ResourceVersion: "22222", Labels: map[string]string{ "app": "example-suspended-1", }, }, - Status: batchv1beta1.CronJobStatus{ - Active: []v1.ObjectReference{}, - LastScheduleTime: &metav1.Time{Time: SuspendedCronJob1LastScheduleTime}, + Status: batchv1.CronJobStatus{ + Active: []v1.ObjectReference{}, + LastScheduleTime: &metav1.Time{Time: SuspendedCronJob1LastScheduleTime}, + LastSuccessfulTime: &metav1.Time{Time: SuspendedCronJob1LastScheduleTime}, }, - Spec: batchv1beta1.CronJobSpec{ - StartingDeadlineSeconds: &StartingDeadlineSeconds300, - ConcurrencyPolicy: "Forbid", - Suspend: &SuspendTrue, - Schedule: "0 */3 * * *", + Spec: batchv1.CronJobSpec{ + StartingDeadlineSeconds: &StartingDeadlineSeconds300, + ConcurrencyPolicy: "Forbid", + Suspend: &SuspendTrue, + Schedule: "0 */3 * * *", + SuccessfulJobsHistoryLimit: &SuccessfulJobHistoryLimit3, + FailedJobsHistoryLimit: &FailedJobHistoryLimit1, }, }, Want: ` - # HELP kube_cronjob_created Unix creation timestamp - # HELP kube_cronjob_info Info about cronjob. - # HELP kube_cronjob_labels Kubernetes labels converted to Prometheus labels. - # HELP kube_cronjob_spec_starting_deadline_seconds Deadline in seconds for starting the job if it misses scheduled time for any reason. - # HELP kube_cronjob_spec_suspend Suspend flag tells the controller to suspend subsequent executions. - # HELP kube_cronjob_status_active Active holds pointers to currently running jobs. - # HELP kube_cronjob_status_last_schedule_time LastScheduleTime keeps information of when was the last time the job was successfully scheduled. + # HELP kube_cronjob_created [STABLE] Unix creation timestamp + # HELP kube_cronjob_info [STABLE] Info about cronjob. + # HELP kube_cronjob_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # HELP kube_cronjob_spec_failed_job_history_limit Failed job history limit tells the controller how many failed jobs should be preserved. + # HELP kube_cronjob_spec_starting_deadline_seconds [STABLE] Deadline in seconds for starting the job if it misses scheduled time for any reason. + # HELP kube_cronjob_spec_successful_job_history_limit Successful job history limit tells the controller how many completed jobs should be preserved. + # HELP kube_cronjob_spec_suspend [STABLE] Suspend flag tells the controller to suspend subsequent executions. + # HELP kube_cronjob_status_active [STABLE] Active holds pointers to currently running jobs. + # HELP kube_cronjob_metadata_resource_version [STABLE] Resource version representing a specific version of the cronjob. + # HELP kube_cronjob_status_last_schedule_time [STABLE] LastScheduleTime keeps information of when was the last time the job was successfully scheduled. + # HELP kube_cronjob_status_last_successful_time LastSuccessfulTime keeps information of when was the last time the job was completed successfully. # TYPE kube_cronjob_created gauge # TYPE kube_cronjob_info gauge # TYPE kube_cronjob_labels gauge + # TYPE kube_cronjob_spec_failed_job_history_limit gauge # TYPE kube_cronjob_spec_starting_deadline_seconds gauge + # TYPE kube_cronjob_spec_successful_job_history_limit gauge # TYPE kube_cronjob_spec_suspend gauge # TYPE kube_cronjob_status_active gauge + # TYPE kube_cronjob_metadata_resource_version gauge # TYPE kube_cronjob_status_last_schedule_time gauge + # TYPE kube_cronjob_status_last_successful_time gauge kube_cronjob_info{concurrency_policy="Forbid",cronjob="SuspendedCronJob1",namespace="ns1",schedule="0 */3 * * *"} 1 - kube_cronjob_labels{cronjob="SuspendedCronJob1",label_app="example-suspended-1",namespace="ns1"} 1 + kube_cronjob_labels{cronjob="SuspendedCronJob1",namespace="ns1"} 1 + kube_cronjob_spec_failed_job_history_limit{cronjob="SuspendedCronJob1",namespace="ns1"} 1 kube_cronjob_spec_starting_deadline_seconds{cronjob="SuspendedCronJob1",namespace="ns1"} 300 + kube_cronjob_spec_successful_job_history_limit{cronjob="SuspendedCronJob1",namespace="ns1"} 3 kube_cronjob_spec_suspend{cronjob="SuspendedCronJob1",namespace="ns1"} 1 kube_cronjob_status_active{cronjob="SuspendedCronJob1",namespace="ns1"} 0 + kube_cronjob_metadata_resource_version{cronjob="SuspendedCronJob1",namespace="ns1"} 22222 kube_cronjob_status_last_schedule_time{cronjob="SuspendedCronJob1",namespace="ns1"} 1.520762696e+09 + kube_cronjob_status_last_successful_time{cronjob="SuspendedCronJob1",namespace="ns1"} 1.520762696e+09 `, - MetricNames: []string{"kube_cronjob_spec_starting_deadline_seconds", "kube_cronjob_status_active", "kube_cronjob_spec_suspend", "kube_cronjob_info", "kube_cronjob_created", "kube_cronjob_labels", "kube_cronjob_status_last_schedule_time"}, + MetricNames: []string{"kube_cronjob_status_last_successful_time", "kube_cronjob_spec_starting_deadline_seconds", "kube_cronjob_status_active", "kube_cronjob_metadata_resource_version", "kube_cronjob_spec_suspend", "kube_cronjob_info", "kube_cronjob_created", "kube_cronjob_labels", "kube_cronjob_status_last_schedule_time", "kube_cronjob_spec_successful_job_history_limit", "kube_cronjob_spec_failed_job_history_limit"}, }, { - Obj: &batchv1beta1.CronJob{ + Obj: &batchv1.CronJob{ ObjectMeta: metav1.ObjectMeta{ Name: "ActiveCronJob1NoLastScheduled", CreationTimestamp: metav1.Time{Time: ActiveCronJob1NoLastScheduledCreationTimestamp}, Namespace: "ns1", Generation: 1, + ResourceVersion: "33333", Labels: map[string]string{ "app": "example-active-no-last-scheduled-1", }, }, - Status: batchv1beta1.CronJobStatus{ - Active: []v1.ObjectReference{}, - LastScheduleTime: nil, + Status: batchv1.CronJobStatus{ + Active: []v1.ObjectReference{}, + LastScheduleTime: nil, + LastSuccessfulTime: nil, }, - Spec: batchv1beta1.CronJobSpec{ - StartingDeadlineSeconds: &StartingDeadlineSeconds300, - ConcurrencyPolicy: "Forbid", - Suspend: &SuspendFalse, - Schedule: "25 * * * *", + Spec: batchv1.CronJobSpec{ + StartingDeadlineSeconds: &StartingDeadlineSeconds300, + ConcurrencyPolicy: "Forbid", + Suspend: &SuspendFalse, + Schedule: "25 * * * *", + SuccessfulJobsHistoryLimit: &SuccessfulJobHistoryLimit3, + FailedJobsHistoryLimit: &FailedJobHistoryLimit1, }, }, Want: ` - # HELP kube_cronjob_created Unix creation timestamp - # HELP kube_cronjob_info Info about cronjob. - # HELP kube_cronjob_labels Kubernetes labels converted to Prometheus labels. - # HELP kube_cronjob_next_schedule_time Next time the cronjob should be scheduled. The time after lastScheduleTime, or after the cron job's creation time if it's never been scheduled. Use this to determine if the job is delayed. - # HELP kube_cronjob_spec_starting_deadline_seconds Deadline in seconds for starting the job if it misses scheduled time for any reason. - # HELP kube_cronjob_spec_suspend Suspend flag tells the controller to suspend subsequent executions. - # HELP kube_cronjob_status_active Active holds pointers to currently running jobs. + # HELP kube_cronjob_created [STABLE] Unix creation timestamp + # HELP kube_cronjob_info [STABLE] Info about cronjob. + # HELP kube_cronjob_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # HELP kube_cronjob_next_schedule_time [STABLE] Next time the cronjob should be scheduled. The time after lastScheduleTime, or after the cron job's creation time if it's never been scheduled. Use this to determine if the job is delayed. + # HELP kube_cronjob_spec_failed_job_history_limit Failed job history limit tells the controller how many failed jobs should be preserved. + # HELP kube_cronjob_spec_starting_deadline_seconds [STABLE] Deadline in seconds for starting the job if it misses scheduled time for any reason. + # HELP kube_cronjob_spec_successful_job_history_limit Successful job history limit tells the controller how many completed jobs should be preserved. + # HELP kube_cronjob_spec_suspend [STABLE] Suspend flag tells the controller to suspend subsequent executions. + # HELP kube_cronjob_status_active [STABLE] Active holds pointers to currently running jobs. + # HELP kube_cronjob_status_last_successful_time LastSuccessfulTime keeps information of when was the last time the job was completed successfully. + # HELP kube_cronjob_metadata_resource_version [STABLE] Resource version representing a specific version of the cronjob. # TYPE kube_cronjob_created gauge # TYPE kube_cronjob_info gauge # TYPE kube_cronjob_labels gauge # TYPE kube_cronjob_next_schedule_time gauge + # TYPE kube_cronjob_spec_failed_job_history_limit gauge # TYPE kube_cronjob_spec_starting_deadline_seconds gauge + # TYPE kube_cronjob_spec_successful_job_history_limit gauge # TYPE kube_cronjob_spec_suspend gauge # TYPE kube_cronjob_status_active gauge + # TYPE kube_cronjob_metadata_resource_version gauge + # TYPE kube_cronjob_status_last_successful_time gauge kube_cronjob_spec_starting_deadline_seconds{cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1"} 300 kube_cronjob_status_active{cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1"} 0 + kube_cronjob_metadata_resource_version{cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1"} 33333 + kube_cronjob_spec_failed_job_history_limit{cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1"} 1 + kube_cronjob_spec_successful_job_history_limit{cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1"} 3 kube_cronjob_spec_suspend{cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1"} 0 kube_cronjob_info{concurrency_policy="Forbid",cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1",schedule="25 * * * *"} 1 kube_cronjob_created{cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1"} 1.520766296e+09 - kube_cronjob_labels{cronjob="ActiveCronJob1NoLastScheduled",label_app="example-active-no-last-scheduled-1",namespace="ns1"} 1 + kube_cronjob_labels{cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1"} 1 ` + fmt.Sprintf("kube_cronjob_next_schedule_time{cronjob=\"ActiveCronJob1NoLastScheduled\",namespace=\"ns1\"} %ve+09\n", float64(ActiveCronJob1NoLastScheduledNextScheduleTime.Unix())/math.Pow10(9)), - MetricNames: []string{"kube_cronjob_next_schedule_time", "kube_cronjob_spec_starting_deadline_seconds", "kube_cronjob_status_active", "kube_cronjob_spec_suspend", "kube_cronjob_info", "kube_cronjob_created", "kube_cronjob_labels"}, + MetricNames: []string{"kube_cronjob_status_last_successful_time", "kube_cronjob_next_schedule_time", "kube_cronjob_spec_starting_deadline_seconds", "kube_cronjob_status_active", "kube_cronjob_metadata_resource_version", "kube_cronjob_spec_suspend", "kube_cronjob_info", "kube_cronjob_created", "kube_cronjob_labels", "kube_cronjob_spec_successful_job_history_limit", "kube_cronjob_spec_failed_job_history_limit"}, }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(cronJobMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(cronJobMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(cronJobMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) + c.Headers = generator.ExtractMetricFamilyHeaders(cronJobMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/daemonset.go b/internal/store/daemonset.go index 7f82627f50..b9d34d7950 100644 --- a/internal/store/daemonset.go +++ b/internal/store/daemonset.go @@ -17,27 +17,37 @@ limitations under the License. package store import ( + "context" + v1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" - "k8s.io/kube-state-metrics/pkg/metric" + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) var ( + descDaemonSetAnnotationsName = "kube_daemonset_annotations" + descDaemonSetAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." descDaemonSetLabelsName = "kube_daemonset_labels" descDaemonSetLabelsHelp = "Kubernetes labels converted to Prometheus labels." descDaemonSetLabelsDefaultLabels = []string{"namespace", "daemonset"} +) - daemonSetMetricFamilies = []metric.FamilyGenerator{ - { - Name: "kube_daemonset_created", - Type: metric.Gauge, - Help: "Unix creation timestamp", - GenerateFunc: wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { +func daemonSetMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_daemonset_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { ms := []*metric.Metric{} if !d.CreationTimestamp.IsZero() { @@ -52,12 +62,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_daemonset_status_current_number_scheduled", - Type: metric.Gauge, - Help: "The number of nodes running at least one daemon pod and are supposed to.", - GenerateFunc: wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_daemonset_status_current_number_scheduled", + "The number of nodes running at least one daemon pod and are supposed to.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -68,12 +80,14 @@ var ( }, } }), - }, - { - Name: "kube_daemonset_status_desired_number_scheduled", - Type: metric.Gauge, - Help: "The number of nodes that should be running the daemon pod.", - GenerateFunc: wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_daemonset_status_desired_number_scheduled", + "The number of nodes that should be running the daemon pod.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -84,12 +98,14 @@ var ( }, } }), - }, - { - Name: "kube_daemonset_status_number_available", - Type: metric.Gauge, - Help: "The number of nodes that should be running the daemon pod and have one or more of the daemon pod running and available", - GenerateFunc: wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_daemonset_status_number_available", + "The number of nodes that should be running the daemon pod and have one or more of the daemon pod running and available", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -100,12 +116,14 @@ var ( }, } }), - }, - { - Name: "kube_daemonset_status_number_misscheduled", - Type: metric.Gauge, - Help: "The number of nodes running a daemon pod but are not supposed to.", - GenerateFunc: wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_daemonset_status_number_misscheduled", + "The number of nodes running a daemon pod but are not supposed to.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -116,12 +134,14 @@ var ( }, } }), - }, - { - Name: "kube_daemonset_status_number_ready", - Type: metric.Gauge, - Help: "The number of nodes that should be running the daemon pod and have one or more of the daemon pod running and ready.", - GenerateFunc: wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_daemonset_status_number_ready", + "The number of nodes that should be running the daemon pod and have one or more of the daemon pod running and ready.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -132,12 +152,14 @@ var ( }, } }), - }, - { - Name: "kube_daemonset_status_number_unavailable", - Type: metric.Gauge, - Help: "The number of nodes that should be running the daemon pod and have none of the daemon pod running and available", - GenerateFunc: wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_daemonset_status_number_unavailable", + "The number of nodes that should be running the daemon pod and have none of the daemon pod running and available", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -148,12 +170,32 @@ var ( }, } }), - }, - { - Name: "kube_daemonset_updated_number_scheduled", - Type: metric.Gauge, - Help: "The total number of nodes that are running updated daemon pod", - GenerateFunc: wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_daemonset_status_observed_generation", + "The most recent generation observed by the daemon set controller.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64(d.Status.ObservedGeneration), + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_daemonset_status_updated_number_scheduled", + "The total number of nodes that are running updated daemon pod", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -162,12 +204,14 @@ var ( }, } }), - }, - { - Name: "kube_daemonset_metadata_generation", - Type: metric.Gauge, - Help: "Sequence number representing a specific generation of the desired state.", - GenerateFunc: wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_daemonset_metadata_generation", + "Sequence number representing a specific generation of the desired state.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -178,13 +222,34 @@ var ( }, } }), - }, - { - Name: descDaemonSetLabelsName, - Type: metric.Gauge, - Help: descDaemonSetLabelsHelp, - GenerateFunc: wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(d.ObjectMeta.Labels) + ), + *generator.NewFamilyGeneratorWithStability( + descDaemonSetAnnotationsName, + descDaemonSetAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", d.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descDaemonSetLabelsName, + descDaemonSetLabelsHelp, + metric.Gauge, + basemetrics.STABLE, + "", + wrapDaemonSetFunc(func(d *v1.DaemonSet) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", d.Labels, allowLabelsList) return &metric.Family{ Metrics: []*metric.Metric{ { @@ -195,9 +260,9 @@ var ( }, } }), - }, + ), } -) +} func wrapDaemonSetFunc(f func(*v1.DaemonSet) *metric.Family) func(interface{}) *metric.Family { return func(obj interface{}) *metric.Family { @@ -206,21 +271,22 @@ func wrapDaemonSetFunc(f func(*v1.DaemonSet) *metric.Family) func(interface{}) * metricFamily := f(daemonSet) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descDaemonSetLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{daemonSet.Namespace, daemonSet.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descDaemonSetLabelsDefaultLabels, []string{daemonSet.Namespace, daemonSet.Name}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createDaemonSetListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createDaemonSetListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.AppsV1().DaemonSets(ns).List(opts) + opts.FieldSelector = fieldSelector + return kubeClient.AppsV1().DaemonSets(ns).List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.AppsV1().DaemonSets(ns).Watch(opts) + opts.FieldSelector = fieldSelector + return kubeClient.AppsV1().DaemonSets(ns).Watch(context.TODO(), opts) }, } } diff --git a/internal/store/daemonset_test.go b/internal/store/daemonset_test.go index 6213a3f771..7a8f06ed3a 100644 --- a/internal/store/daemonset_test.go +++ b/internal/store/daemonset_test.go @@ -23,12 +23,15 @@ import ( v1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) func TestDaemonSetStore(t *testing.T) { cases := []generateMetricsTestCase{ { + AllowAnnotationsList: []string{ + "app.k8s.io/owner", + }, Obj: &v1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: "ds1", @@ -36,6 +39,10 @@ func TestDaemonSetStore(t *testing.T) { Labels: map[string]string{ "app": "example1", }, + Annotations: map[string]string{ + "app": "mysql-server", + "app.k8s.io/owner": "@foo", + }, Generation: 21, }, Status: v1.DaemonSetStatus{ @@ -43,18 +50,22 @@ func TestDaemonSetStore(t *testing.T) { NumberMisscheduled: 10, DesiredNumberScheduled: 5, NumberReady: 5, + ObservedGeneration: 2, }, }, Want: ` - # HELP kube_daemonset_labels Kubernetes labels converted to Prometheus labels. - # HELP kube_daemonset_metadata_generation Sequence number representing a specific generation of the desired state. - # HELP kube_daemonset_status_current_number_scheduled The number of nodes running at least one daemon pod and are supposed to. - # HELP kube_daemonset_status_desired_number_scheduled The number of nodes that should be running the daemon pod. - # HELP kube_daemonset_status_number_available The number of nodes that should be running the daemon pod and have one or more of the daemon pod running and available - # HELP kube_daemonset_status_number_misscheduled The number of nodes running a daemon pod but are not supposed to. - # HELP kube_daemonset_status_number_ready The number of nodes that should be running the daemon pod and have one or more of the daemon pod running and ready. - # HELP kube_daemonset_status_number_unavailable The number of nodes that should be running the daemon pod and have none of the daemon pod running and available - # HELP kube_daemonset_updated_number_scheduled The total number of nodes that are running updated daemon pod + # HELP kube_daemonset_annotations Kubernetes annotations converted to Prometheus labels. + # HELP kube_daemonset_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # HELP kube_daemonset_metadata_generation [STABLE] Sequence number representing a specific generation of the desired state. + # HELP kube_daemonset_status_current_number_scheduled [STABLE] The number of nodes running at least one daemon pod and are supposed to. + # HELP kube_daemonset_status_desired_number_scheduled [STABLE] The number of nodes that should be running the daemon pod. + # HELP kube_daemonset_status_number_available [STABLE] The number of nodes that should be running the daemon pod and have one or more of the daemon pod running and available + # HELP kube_daemonset_status_number_misscheduled [STABLE] The number of nodes running a daemon pod but are not supposed to. + # HELP kube_daemonset_status_number_ready [STABLE] The number of nodes that should be running the daemon pod and have one or more of the daemon pod running and ready. + # HELP kube_daemonset_status_number_unavailable [STABLE] The number of nodes that should be running the daemon pod and have none of the daemon pod running and available + # HELP kube_daemonset_status_observed_generation [STABLE] The most recent generation observed by the daemon set controller. + # HELP kube_daemonset_status_updated_number_scheduled [STABLE] The total number of nodes that are running updated daemon pod + # TYPE kube_daemonset_annotations gauge # TYPE kube_daemonset_labels gauge # TYPE kube_daemonset_metadata_generation gauge # TYPE kube_daemonset_status_current_number_scheduled gauge @@ -63,7 +74,8 @@ func TestDaemonSetStore(t *testing.T) { # TYPE kube_daemonset_status_number_misscheduled gauge # TYPE kube_daemonset_status_number_ready gauge # TYPE kube_daemonset_status_number_unavailable gauge - # TYPE kube_daemonset_updated_number_scheduled gauge + # TYPE kube_daemonset_status_observed_generation gauge + # TYPE kube_daemonset_status_updated_number_scheduled gauge kube_daemonset_metadata_generation{daemonset="ds1",namespace="ns1"} 21 kube_daemonset_status_current_number_scheduled{daemonset="ds1",namespace="ns1"} 15 kube_daemonset_status_desired_number_scheduled{daemonset="ds1",namespace="ns1"} 5 @@ -71,10 +83,13 @@ func TestDaemonSetStore(t *testing.T) { kube_daemonset_status_number_misscheduled{daemonset="ds1",namespace="ns1"} 10 kube_daemonset_status_number_ready{daemonset="ds1",namespace="ns1"} 5 kube_daemonset_status_number_unavailable{daemonset="ds1",namespace="ns1"} 0 - kube_daemonset_updated_number_scheduled{daemonset="ds1",namespace="ns1"} 0 - kube_daemonset_labels{daemonset="ds1",label_app="example1",namespace="ns1"} 1 + kube_daemonset_status_observed_generation{daemonset="ds1",namespace="ns1"} 2 + kube_daemonset_status_updated_number_scheduled{daemonset="ds1",namespace="ns1"} 0 + kube_daemonset_annotations{annotation_app_k8s_io_owner="@foo",daemonset="ds1",namespace="ns1"} 1 + kube_daemonset_labels{daemonset="ds1",namespace="ns1"} 1 `, MetricNames: []string{ + "kube_daemonset_annotations", "kube_daemonset_labels", "kube_daemonset_metadata_generation", "kube_daemonset_status_current_number_scheduled", @@ -83,7 +98,8 @@ func TestDaemonSetStore(t *testing.T) { "kube_daemonset_status_number_misscheduled", "kube_daemonset_status_number_ready", "kube_daemonset_status_number_unavailable", - "kube_daemonset_updated_number_scheduled", + "kube_daemonset_status_observed_generation", + "kube_daemonset_status_updated_number_scheduled", }, }, { @@ -105,25 +121,25 @@ func TestDaemonSetStore(t *testing.T) { }, }, Want: ` - # HELP kube_daemonset_created Unix creation timestamp + # HELP kube_daemonset_created [STABLE] Unix creation timestamp # TYPE kube_daemonset_created gauge - # HELP kube_daemonset_status_current_number_scheduled The number of nodes running at least one daemon pod and are supposed to. + # HELP kube_daemonset_status_current_number_scheduled [STABLE] The number of nodes running at least one daemon pod and are supposed to. # TYPE kube_daemonset_status_current_number_scheduled gauge - # HELP kube_daemonset_status_desired_number_scheduled The number of nodes that should be running the daemon pod. + # HELP kube_daemonset_status_desired_number_scheduled [STABLE] The number of nodes that should be running the daemon pod. # TYPE kube_daemonset_status_desired_number_scheduled gauge - # HELP kube_daemonset_status_number_available The number of nodes that should be running the daemon pod and have one or more of the daemon pod running and available + # HELP kube_daemonset_status_number_available [STABLE] The number of nodes that should be running the daemon pod and have one or more of the daemon pod running and available # TYPE kube_daemonset_status_number_available gauge - # HELP kube_daemonset_status_number_misscheduled The number of nodes running a daemon pod but are not supposed to. + # HELP kube_daemonset_status_number_misscheduled [STABLE] The number of nodes running a daemon pod but are not supposed to. # TYPE kube_daemonset_status_number_misscheduled gauge - # HELP kube_daemonset_status_number_ready The number of nodes that should be running the daemon pod and have one or more of the daemon pod running and ready. + # HELP kube_daemonset_status_number_ready [STABLE] The number of nodes that should be running the daemon pod and have one or more of the daemon pod running and ready. # TYPE kube_daemonset_status_number_ready gauge - # HELP kube_daemonset_status_number_unavailable The number of nodes that should be running the daemon pod and have none of the daemon pod running and available + # HELP kube_daemonset_status_number_unavailable [STABLE] The number of nodes that should be running the daemon pod and have none of the daemon pod running and available # TYPE kube_daemonset_status_number_unavailable gauge - # HELP kube_daemonset_updated_number_scheduled The total number of nodes that are running updated daemon pod - # TYPE kube_daemonset_updated_number_scheduled gauge - # HELP kube_daemonset_metadata_generation Sequence number representing a specific generation of the desired state. + # HELP kube_daemonset_status_updated_number_scheduled [STABLE] The total number of nodes that are running updated daemon pod + # TYPE kube_daemonset_status_updated_number_scheduled gauge + # HELP kube_daemonset_metadata_generation [STABLE] Sequence number representing a specific generation of the desired state. # TYPE kube_daemonset_metadata_generation gauge - # HELP kube_daemonset_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_daemonset_labels [STABLE] Kubernetes labels converted to Prometheus labels. # TYPE kube_daemonset_labels gauge kube_daemonset_metadata_generation{daemonset="ds2",namespace="ns2"} 14 kube_daemonset_status_current_number_scheduled{daemonset="ds2",namespace="ns2"} 10 @@ -132,8 +148,8 @@ func TestDaemonSetStore(t *testing.T) { kube_daemonset_status_number_misscheduled{daemonset="ds2",namespace="ns2"} 5 kube_daemonset_status_number_ready{daemonset="ds2",namespace="ns2"} 0 kube_daemonset_status_number_unavailable{daemonset="ds2",namespace="ns2"} 0 - kube_daemonset_updated_number_scheduled{daemonset="ds2",namespace="ns2"} 0 - kube_daemonset_labels{daemonset="ds2",label_app="example2",namespace="ns2"} 1 + kube_daemonset_status_updated_number_scheduled{daemonset="ds2",namespace="ns2"} 0 + kube_daemonset_labels{daemonset="ds2",namespace="ns2"} 1 kube_daemonset_created{namespace="ns2",daemonset="ds2"} 1.5e+09 `, MetricNames: []string{ @@ -146,7 +162,7 @@ func TestDaemonSetStore(t *testing.T) { "kube_daemonset_status_number_misscheduled", "kube_daemonset_status_number_ready", "kube_daemonset_status_number_unavailable", - "kube_daemonset_updated_number_scheduled", + "kube_daemonset_status_updated_number_scheduled", }, }, { @@ -171,25 +187,25 @@ func TestDaemonSetStore(t *testing.T) { }, }, Want: ` - # HELP kube_daemonset_created Unix creation timestamp + # HELP kube_daemonset_created [STABLE] Unix creation timestamp # TYPE kube_daemonset_created gauge - # HELP kube_daemonset_status_current_number_scheduled The number of nodes running at least one daemon pod and are supposed to. + # HELP kube_daemonset_status_current_number_scheduled [STABLE] The number of nodes running at least one daemon pod and are supposed to. # TYPE kube_daemonset_status_current_number_scheduled gauge - # HELP kube_daemonset_status_desired_number_scheduled The number of nodes that should be running the daemon pod. + # HELP kube_daemonset_status_desired_number_scheduled [STABLE] The number of nodes that should be running the daemon pod. # TYPE kube_daemonset_status_desired_number_scheduled gauge - # HELP kube_daemonset_status_number_available The number of nodes that should be running the daemon pod and have one or more of the daemon pod running and available + # HELP kube_daemonset_status_number_available [STABLE] The number of nodes that should be running the daemon pod and have one or more of the daemon pod running and available # TYPE kube_daemonset_status_number_available gauge - # HELP kube_daemonset_status_number_misscheduled The number of nodes running a daemon pod but are not supposed to. + # HELP kube_daemonset_status_number_misscheduled [STABLE] The number of nodes running a daemon pod but are not supposed to. # TYPE kube_daemonset_status_number_misscheduled gauge - # HELP kube_daemonset_status_number_ready The number of nodes that should be running the daemon pod and have one or more of the daemon pod running and ready. + # HELP kube_daemonset_status_number_ready [STABLE] The number of nodes that should be running the daemon pod and have one or more of the daemon pod running and ready. # TYPE kube_daemonset_status_number_ready gauge - # HELP kube_daemonset_status_number_unavailable The number of nodes that should be running the daemon pod and have none of the daemon pod running and available + # HELP kube_daemonset_status_number_unavailable [STABLE] The number of nodes that should be running the daemon pod and have none of the daemon pod running and available # TYPE kube_daemonset_status_number_unavailable gauge - # HELP kube_daemonset_updated_number_scheduled The total number of nodes that are running updated daemon pod - # TYPE kube_daemonset_updated_number_scheduled gauge - # HELP kube_daemonset_metadata_generation Sequence number representing a specific generation of the desired state. + # HELP kube_daemonset_status_updated_number_scheduled [STABLE] The total number of nodes that are running updated daemon pod + # TYPE kube_daemonset_status_updated_number_scheduled gauge + # HELP kube_daemonset_metadata_generation [STABLE] Sequence number representing a specific generation of the desired state. # TYPE kube_daemonset_metadata_generation gauge - # HELP kube_daemonset_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_daemonset_labels [STABLE] Kubernetes labels converted to Prometheus labels. # TYPE kube_daemonset_labels gauge kube_daemonset_created{daemonset="ds3",namespace="ns3"} 1.5e+09 kube_daemonset_metadata_generation{daemonset="ds3",namespace="ns3"} 15 @@ -199,8 +215,8 @@ func TestDaemonSetStore(t *testing.T) { kube_daemonset_status_number_misscheduled{daemonset="ds3",namespace="ns3"} 5 kube_daemonset_status_number_ready{daemonset="ds3",namespace="ns3"} 5 kube_daemonset_status_number_unavailable{daemonset="ds3",namespace="ns3"} 5 - kube_daemonset_updated_number_scheduled{daemonset="ds3",namespace="ns3"} 5 - kube_daemonset_labels{daemonset="ds3",label_app="example3",namespace="ns3"} 1 + kube_daemonset_status_updated_number_scheduled{daemonset="ds3",namespace="ns3"} 5 + kube_daemonset_labels{daemonset="ds3",namespace="ns3"} 1 `, MetricNames: []string{ "kube_daemonset_created", @@ -212,13 +228,13 @@ func TestDaemonSetStore(t *testing.T) { "kube_daemonset_status_number_misscheduled", "kube_daemonset_status_number_ready", "kube_daemonset_status_number_unavailable", - "kube_daemonset_updated_number_scheduled", + "kube_daemonset_status_updated_number_scheduled", }, }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(daemonSetMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(daemonSetMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(daemonSetMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) + c.Headers = generator.ExtractMetricFamilyHeaders(daemonSetMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/deployment.go b/internal/store/deployment.go index b42f0545c7..0535c3b87a 100644 --- a/internal/store/deployment.go +++ b/internal/store/deployment.go @@ -17,7 +17,12 @@ limitations under the License. package store import ( - "k8s.io/kube-state-metrics/pkg/metric" + "context" + + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" v1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -29,16 +34,22 @@ import ( ) var ( + descDeploymentAnnotationsName = "kube_deployment_annotations" + descDeploymentAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." descDeploymentLabelsName = "kube_deployment_labels" descDeploymentLabelsHelp = "Kubernetes labels converted to Prometheus labels." descDeploymentLabelsDefaultLabels = []string{"namespace", "deployment"} +) - deploymentMetricFamilies = []metric.FamilyGenerator{ - { - Name: "kube_deployment_created", - Type: metric.Gauge, - Help: "Unix creation timestamp", - GenerateFunc: wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { +func deploymentMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_deployment_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { ms := []*metric.Metric{} if !d.CreationTimestamp.IsZero() { @@ -51,12 +62,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_deployment_status_replicas", - Type: metric.Gauge, - Help: "The number of replicas per deployment.", - GenerateFunc: wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_deployment_status_replicas", + "The number of replicas per deployment.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -65,12 +78,30 @@ var ( }, } }), - }, - { - Name: "kube_deployment_status_replicas_available", - Type: metric.Gauge, - Help: "The number of available replicas per deployment.", - GenerateFunc: wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_deployment_status_replicas_ready", + "The number of ready replicas per deployment.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { + return &metric.Family{ + Metrics: []*metric.Metric{ + { + Value: float64(d.Status.ReadyReplicas), + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_deployment_status_replicas_available", + "The number of available replicas per deployment.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -79,12 +110,14 @@ var ( }, } }), - }, - { - Name: "kube_deployment_status_replicas_unavailable", - Type: metric.Gauge, - Help: "The number of unavailable replicas per deployment.", - GenerateFunc: wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_deployment_status_replicas_unavailable", + "The number of unavailable replicas per deployment.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -93,12 +126,14 @@ var ( }, } }), - }, - { - Name: "kube_deployment_status_replicas_updated", - Type: metric.Gauge, - Help: "The number of updated replicas per deployment.", - GenerateFunc: wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_deployment_status_replicas_updated", + "The number of updated replicas per deployment.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -107,12 +142,14 @@ var ( }, } }), - }, - { - Name: "kube_deployment_status_observed_generation", - Type: metric.Gauge, - Help: "The generation observed by the deployment controller.", - GenerateFunc: wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_deployment_status_observed_generation", + "The generation observed by the deployment controller.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -121,12 +158,14 @@ var ( }, } }), - }, - { - Name: "kube_deployment_status_condition", - Type: metric.Gauge, - Help: "The current status conditions of a deployment.", - GenerateFunc: wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_deployment_status_condition", + "The current status conditions of a deployment.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { ms := make([]*metric.Metric, len(d.Status.Conditions)*len(conditionStatuses)) for i, c := range d.Status.Conditions { @@ -145,12 +184,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_deployment_spec_replicas", - Type: metric.Gauge, - Help: "Number of desired pods for a deployment.", - GenerateFunc: wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_deployment_spec_replicas", + "Number of desired pods for a deployment.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -159,12 +200,14 @@ var ( }, } }), - }, - { - Name: "kube_deployment_spec_paused", - Type: metric.Gauge, - Help: "Whether the deployment is paused and will not be processed by the deployment controller.", - GenerateFunc: wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_deployment_spec_paused", + "Whether the deployment is paused and will not be processed by the deployment controller.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -173,17 +216,19 @@ var ( }, } }), - }, - { - Name: "kube_deployment_spec_strategy_rollingupdate_max_unavailable", - Type: metric.Gauge, - Help: "Maximum number of unavailable replicas during a rolling update of a deployment.", - GenerateFunc: wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_deployment_spec_strategy_rollingupdate_max_unavailable", + "Maximum number of unavailable replicas during a rolling update of a deployment.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { if d.Spec.Strategy.RollingUpdate == nil { return &metric.Family{} } - maxUnavailable, err := intstr.GetValueFromIntOrPercent(d.Spec.Strategy.RollingUpdate.MaxUnavailable, int(*d.Spec.Replicas), true) + maxUnavailable, err := intstr.GetScaledValueFromIntOrPercent(d.Spec.Strategy.RollingUpdate.MaxUnavailable, int(*d.Spec.Replicas), false) if err != nil { panic(err) } @@ -196,17 +241,19 @@ var ( }, } }), - }, - { - Name: "kube_deployment_spec_strategy_rollingupdate_max_surge", - Type: metric.Gauge, - Help: "Maximum number of replicas that can be scheduled above the desired number of replicas during a rolling update of a deployment.", - GenerateFunc: wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_deployment_spec_strategy_rollingupdate_max_surge", + "Maximum number of replicas that can be scheduled above the desired number of replicas during a rolling update of a deployment.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { if d.Spec.Strategy.RollingUpdate == nil { return &metric.Family{} } - maxSurge, err := intstr.GetValueFromIntOrPercent(d.Spec.Strategy.RollingUpdate.MaxSurge, int(*d.Spec.Replicas), true) + maxSurge, err := intstr.GetScaledValueFromIntOrPercent(d.Spec.Strategy.RollingUpdate.MaxSurge, int(*d.Spec.Replicas), true) if err != nil { panic(err) } @@ -219,12 +266,14 @@ var ( }, } }), - }, - { - Name: "kube_deployment_metadata_generation", - Type: metric.Gauge, - Help: "Sequence number representing a specific generation of the desired state.", - GenerateFunc: wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_deployment_metadata_generation", + "Sequence number representing a specific generation of the desired state.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -233,13 +282,34 @@ var ( }, } }), - }, - { - Name: descDeploymentLabelsName, - Type: metric.Gauge, - Help: descDeploymentLabelsHelp, - GenerateFunc: wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(d.Labels) + ), + *generator.NewFamilyGeneratorWithStability( + descDeploymentAnnotationsName, + descDeploymentAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", d.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descDeploymentLabelsName, + descDeploymentLabelsHelp, + metric.Gauge, + basemetrics.STABLE, + "", + wrapDeploymentFunc(func(d *v1.Deployment) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", d.Labels, allowLabelsList) return &metric.Family{ Metrics: []*metric.Metric{ { @@ -250,9 +320,9 @@ var ( }, } }), - }, + ), } -) +} func wrapDeploymentFunc(f func(*v1.Deployment) *metric.Family) func(interface{}) *metric.Family { return func(obj interface{}) *metric.Family { @@ -261,21 +331,22 @@ func wrapDeploymentFunc(f func(*v1.Deployment) *metric.Family) func(interface{}) metricFamily := f(deployment) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descDeploymentLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{deployment.Namespace, deployment.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descDeploymentLabelsDefaultLabels, []string{deployment.Namespace, deployment.Name}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createDeploymentListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createDeploymentListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.AppsV1().Deployments(ns).List(opts) + opts.FieldSelector = fieldSelector + return kubeClient.AppsV1().Deployments(ns).List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.AppsV1().Deployments(ns).Watch(opts) + opts.FieldSelector = fieldSelector + return kubeClient.AppsV1().Deployments(ns).Watch(context.TODO(), opts) }, } } diff --git a/internal/store/deployment_test.go b/internal/store/deployment_test.go index 0f52b652ae..0223011e09 100644 --- a/internal/store/deployment_test.go +++ b/internal/store/deployment_test.go @@ -25,7 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) var ( @@ -33,7 +33,7 @@ var ( depl2Replicas int32 = 5 depl1MaxUnavailable = intstr.FromInt(10) - depl2MaxUnavailable = intstr.FromString("20%") + depl2MaxUnavailable = intstr.FromString("25%") depl1MaxSurge = intstr.FromInt(10) depl2MaxSurge = intstr.FromString("20%") @@ -43,40 +43,48 @@ func TestDeploymentStore(t *testing.T) { // Fixed metadata on type and help text. We prepend this to every expected // output so we only have to modify a single place when doing adjustments. const metadata = ` - # HELP kube_deployment_created Unix creation timestamp + # HELP kube_deployment_annotations Kubernetes annotations converted to Prometheus labels. + # TYPE kube_deployment_annotations gauge + # HELP kube_deployment_created [STABLE] Unix creation timestamp # TYPE kube_deployment_created gauge - # HELP kube_deployment_metadata_generation Sequence number representing a specific generation of the desired state. + # HELP kube_deployment_metadata_generation [STABLE] Sequence number representing a specific generation of the desired state. # TYPE kube_deployment_metadata_generation gauge - # HELP kube_deployment_spec_paused Whether the deployment is paused and will not be processed by the deployment controller. + # HELP kube_deployment_spec_paused [STABLE] Whether the deployment is paused and will not be processed by the deployment controller. # TYPE kube_deployment_spec_paused gauge - # HELP kube_deployment_spec_replicas Number of desired pods for a deployment. + # HELP kube_deployment_spec_replicas [STABLE] Number of desired pods for a deployment. # TYPE kube_deployment_spec_replicas gauge - # HELP kube_deployment_status_replicas The number of replicas per deployment. + # HELP kube_deployment_status_replicas [STABLE] The number of replicas per deployment. # TYPE kube_deployment_status_replicas gauge - # HELP kube_deployment_status_replicas_available The number of available replicas per deployment. + # HELP kube_deployment_status_replicas_ready [STABLE] The number of ready replicas per deployment. + # TYPE kube_deployment_status_replicas_ready gauge + # HELP kube_deployment_status_replicas_available [STABLE] The number of available replicas per deployment. # TYPE kube_deployment_status_replicas_available gauge - # HELP kube_deployment_status_replicas_unavailable The number of unavailable replicas per deployment. + # HELP kube_deployment_status_replicas_unavailable [STABLE] The number of unavailable replicas per deployment. # TYPE kube_deployment_status_replicas_unavailable gauge - # HELP kube_deployment_status_replicas_updated The number of updated replicas per deployment. + # HELP kube_deployment_status_replicas_updated [STABLE] The number of updated replicas per deployment. # TYPE kube_deployment_status_replicas_updated gauge - # HELP kube_deployment_status_observed_generation The generation observed by the deployment controller. + # HELP kube_deployment_status_observed_generation [STABLE] The generation observed by the deployment controller. # TYPE kube_deployment_status_observed_generation gauge - # HELP kube_deployment_status_condition The current status conditions of a deployment. + # HELP kube_deployment_status_condition [STABLE] The current status conditions of a deployment. # TYPE kube_deployment_status_condition gauge - # HELP kube_deployment_spec_strategy_rollingupdate_max_unavailable Maximum number of unavailable replicas during a rolling update of a deployment. + # HELP kube_deployment_spec_strategy_rollingupdate_max_unavailable [STABLE] Maximum number of unavailable replicas during a rolling update of a deployment. # TYPE kube_deployment_spec_strategy_rollingupdate_max_unavailable gauge - # HELP kube_deployment_spec_strategy_rollingupdate_max_surge Maximum number of replicas that can be scheduled above the desired number of replicas during a rolling update of a deployment. + # HELP kube_deployment_spec_strategy_rollingupdate_max_surge [STABLE] Maximum number of replicas that can be scheduled above the desired number of replicas during a rolling update of a deployment. # TYPE kube_deployment_spec_strategy_rollingupdate_max_surge gauge - # HELP kube_deployment_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_deployment_labels [STABLE] Kubernetes labels converted to Prometheus labels. # TYPE kube_deployment_labels gauge ` cases := []generateMetricsTestCase{ { + AllowAnnotationsList: []string{"company.io/team"}, Obj: &v1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "depl1", CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, Namespace: "ns1", + Annotations: map[string]string{ + "company.io/team": "my-brilliant-team", + }, Labels: map[string]string{ "app": "example1", }, @@ -84,6 +92,7 @@ func TestDeploymentStore(t *testing.T) { }, Status: v1.DeploymentStatus{ Replicas: 15, + ReadyReplicas: 10, AvailableReplicas: 10, UnavailableReplicas: 5, UpdatedReplicas: 2, @@ -104,8 +113,9 @@ func TestDeploymentStore(t *testing.T) { }, }, Want: metadata + ` + kube_deployment_annotations{annotation_company_io_team="my-brilliant-team",deployment="depl1",namespace="ns1"} 1 kube_deployment_created{deployment="depl1",namespace="ns1"} 1.5e+09 - kube_deployment_labels{deployment="depl1",label_app="example1",namespace="ns1"} 1 + kube_deployment_labels{deployment="depl1",namespace="ns1"} 1 kube_deployment_metadata_generation{deployment="depl1",namespace="ns1"} 21 kube_deployment_spec_paused{deployment="depl1",namespace="ns1"} 0 kube_deployment_spec_replicas{deployment="depl1",namespace="ns1"} 200 @@ -116,6 +126,7 @@ func TestDeploymentStore(t *testing.T) { kube_deployment_status_replicas_unavailable{deployment="depl1",namespace="ns1"} 5 kube_deployment_status_replicas_updated{deployment="depl1",namespace="ns1"} 2 kube_deployment_status_replicas{deployment="depl1",namespace="ns1"} 15 + kube_deployment_status_replicas_ready{deployment="depl1",namespace="ns1"} 10 kube_deployment_status_condition{deployment="depl1",namespace="ns1",condition="Available",status="true"} 1 kube_deployment_status_condition{deployment="depl1",namespace="ns1",condition="Progressing",status="true"} 1 kube_deployment_status_condition{deployment="depl1",namespace="ns1",condition="Available",status="false"} 0 @@ -136,6 +147,7 @@ func TestDeploymentStore(t *testing.T) { }, Status: v1.DeploymentStatus{ Replicas: 10, + ReadyReplicas: 5, AvailableReplicas: 5, UnavailableReplicas: 0, UpdatedReplicas: 1, @@ -158,7 +170,8 @@ func TestDeploymentStore(t *testing.T) { }, }, Want: metadata + ` - kube_deployment_labels{deployment="depl2",label_app="example2",namespace="ns2"} 1 + kube_deployment_annotations{deployment="depl2",namespace="ns2"} 1 + kube_deployment_labels{deployment="depl2",namespace="ns2"} 1 kube_deployment_metadata_generation{deployment="depl2",namespace="ns2"} 14 kube_deployment_spec_paused{deployment="depl2",namespace="ns2"} 1 kube_deployment_spec_replicas{deployment="depl2",namespace="ns2"} 5 @@ -169,6 +182,7 @@ func TestDeploymentStore(t *testing.T) { kube_deployment_status_replicas_unavailable{deployment="depl2",namespace="ns2"} 0 kube_deployment_status_replicas_updated{deployment="depl2",namespace="ns2"} 1 kube_deployment_status_replicas{deployment="depl2",namespace="ns2"} 10 + kube_deployment_status_replicas_ready{deployment="depl2",namespace="ns2"} 5 kube_deployment_status_condition{deployment="depl2",namespace="ns2",condition="Available",status="true"} 0 kube_deployment_status_condition{deployment="depl2",namespace="ns2",condition="Progressing",status="true"} 0 kube_deployment_status_condition{deployment="depl2",namespace="ns2",condition="ReplicaFailure",status="true"} 1 @@ -183,8 +197,8 @@ func TestDeploymentStore(t *testing.T) { } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(deploymentMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(deploymentMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(deploymentMetricFamilies(c.AllowAnnotationsList, nil)) + c.Headers = generator.ExtractMetricFamilyHeaders(deploymentMetricFamilies(c.AllowAnnotationsList, nil)) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/endpoint.go b/internal/store/endpoint.go index d48748b941..cd06d00fe8 100644 --- a/internal/store/endpoint.go +++ b/internal/store/endpoint.go @@ -17,27 +17,38 @@ limitations under the License. package store import ( + "context" + "strconv" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" - "k8s.io/kube-state-metrics/pkg/metric" + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) var ( + descEndpointAnnotationsName = "kube_endpoint_annotations" + descEndpointAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." descEndpointLabelsName = "kube_endpoint_labels" descEndpointLabelsHelp = "Kubernetes labels converted to Prometheus labels." descEndpointLabelsDefaultLabels = []string{"namespace", "endpoint"} +) - endpointMetricFamilies = []metric.FamilyGenerator{ - { - Name: "kube_endpoint_info", - Type: metric.Gauge, - Help: "Information about endpoint.", - GenerateFunc: wrapEndpointFunc(func(e *v1.Endpoints) *metric.Family { +func endpointMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_endpoint_info", + "Information about endpoint.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapEndpointFunc(func(e *v1.Endpoints) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -46,12 +57,14 @@ var ( }, } }), - }, - { - Name: "kube_endpoint_created", - Type: metric.Gauge, - Help: "Unix creation timestamp", - GenerateFunc: wrapEndpointFunc(func(e *v1.Endpoints) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_endpoint_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.STABLE, + "", + wrapEndpointFunc(func(e *v1.Endpoints) *metric.Family { ms := []*metric.Metric{} if !e.CreationTimestamp.IsZero() { @@ -65,13 +78,34 @@ var ( Metrics: ms, } }), - }, - { - Name: descEndpointLabelsName, - Type: metric.Gauge, - Help: descEndpointLabelsHelp, - GenerateFunc: wrapEndpointFunc(func(e *v1.Endpoints) *metric.Family { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(e.Labels) + ), + *generator.NewFamilyGeneratorWithStability( + descEndpointAnnotationsName, + descEndpointAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapEndpointFunc(func(e *v1.Endpoints) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", e.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descEndpointLabelsName, + descEndpointLabelsHelp, + metric.Gauge, + basemetrics.STABLE, + "", + wrapEndpointFunc(func(e *v1.Endpoints) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", e.Labels, allowLabelsList) return &metric.Family{ Metrics: []*metric.Metric{ { @@ -82,12 +116,14 @@ var ( }, } }), - }, - { - Name: "kube_endpoint_address_available", - Type: metric.Gauge, - Help: "Number of addresses available in endpoint.", - GenerateFunc: wrapEndpointFunc(func(e *v1.Endpoints) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_endpoint_address_available", + "Number of addresses available in endpoint.", + metric.Gauge, + basemetrics.ALPHA, + "v2.6.0", + wrapEndpointFunc(func(e *v1.Endpoints) *metric.Family { var available int for _, s := range e.Subsets { available += len(s.Addresses) * len(s.Ports) @@ -101,12 +137,14 @@ var ( }, } }), - }, - { - Name: "kube_endpoint_address_not_ready", - Type: metric.Gauge, - Help: "Number of addresses not ready in endpoint", - GenerateFunc: wrapEndpointFunc(func(e *v1.Endpoints) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_endpoint_address_not_ready", + "Number of addresses not ready in endpoint", + metric.Gauge, + basemetrics.ALPHA, + "v2.6.0", + wrapEndpointFunc(func(e *v1.Endpoints) *metric.Family { var notReady int for _, s := range e.Subsets { notReady += len(s.NotReadyAddresses) * len(s.Ports) @@ -119,9 +157,60 @@ var ( }, } }), - }, + ), + *generator.NewFamilyGeneratorWithStability( + "kube_endpoint_address", + "Information about Endpoint available and non available addresses.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapEndpointFunc(func(e *v1.Endpoints) *metric.Family { + ms := []*metric.Metric{} + for _, s := range e.Subsets { + for _, available := range s.Addresses { + ms = append(ms, &metric.Metric{ + LabelValues: []string{available.IP, "true"}, + LabelKeys: []string{"ip", "ready"}, + Value: 1, + }) + } + for _, notReadyAddresses := range s.NotReadyAddresses { + ms = append(ms, &metric.Metric{ + LabelValues: []string{notReadyAddresses.IP, "false"}, + LabelKeys: []string{"ip", "ready"}, + Value: 1, + }) + } + } + return &metric.Family{ + Metrics: ms, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_endpoint_ports", + "Information about the Endpoint ports.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapEndpointFunc(func(e *v1.Endpoints) *metric.Family { + ms := []*metric.Metric{} + for _, s := range e.Subsets { + for _, port := range s.Ports { + ms = append(ms, &metric.Metric{ + LabelValues: []string{port.Name, string(port.Protocol), strconv.FormatInt(int64(port.Port), 10)}, + LabelKeys: []string{"port_name", "port_protocol", "port_number"}, + Value: 1, + }) + } + } + return &metric.Family{ + Metrics: ms, + } + }), + ), } -) +} func wrapEndpointFunc(f func(*v1.Endpoints) *metric.Family) func(interface{}) *metric.Family { return func(obj interface{}) *metric.Family { @@ -130,21 +219,22 @@ func wrapEndpointFunc(f func(*v1.Endpoints) *metric.Family) func(interface{}) *m metricFamily := f(endpoint) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descEndpointLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{endpoint.Namespace, endpoint.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descEndpointLabelsDefaultLabels, []string{endpoint.Namespace, endpoint.Name}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createEndpointsListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createEndpointsListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.CoreV1().Endpoints(ns).List(opts) + opts.FieldSelector = fieldSelector + return kubeClient.CoreV1().Endpoints(ns).List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.CoreV1().Endpoints(ns).Watch(opts) + opts.FieldSelector = fieldSelector + return kubeClient.CoreV1().Endpoints(ns).Watch(context.TODO(), opts) }, } } diff --git a/internal/store/endpoint_test.go b/internal/store/endpoint_test.go index 97493994e4..46df12e7df 100644 --- a/internal/store/endpoint_test.go +++ b/internal/store/endpoint_test.go @@ -23,23 +23,29 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) func TestEndpointStore(t *testing.T) { // Fixed metadata on type and help text. We prepend this to every expected // output so we only have to modify a single place when doing adjustments. const metadata = ` - # HELP kube_endpoint_address_available Number of addresses available in endpoint. + # HELP kube_endpoint_annotations Kubernetes annotations converted to Prometheus labels. + # TYPE kube_endpoint_annotations gauge + # HELP kube_endpoint_address_available (Deprecated since v2.6.0) Number of addresses available in endpoint. # TYPE kube_endpoint_address_available gauge - # HELP kube_endpoint_address_not_ready Number of addresses not ready in endpoint + # HELP kube_endpoint_address_not_ready (Deprecated since v2.6.0) Number of addresses not ready in endpoint # TYPE kube_endpoint_address_not_ready gauge - # HELP kube_endpoint_created Unix creation timestamp + # HELP kube_endpoint_created [STABLE] Unix creation timestamp # TYPE kube_endpoint_created gauge - # HELP kube_endpoint_info Information about endpoint. + # HELP kube_endpoint_info [STABLE] Information about endpoint. # TYPE kube_endpoint_info gauge - # HELP kube_endpoint_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_endpoint_labels [STABLE] Kubernetes labels converted to Prometheus labels. # TYPE kube_endpoint_labels gauge + # HELP kube_endpoint_ports [STABLE] Information about the Endpoint ports. + # TYPE kube_endpoint_ports gauge + # HELP kube_endpoint_address [STABLE] Information about Endpoint available and non available addresses. + # TYPE kube_endpoint_address gauge ` cases := []generateMetricsTestCase{ { @@ -53,32 +59,164 @@ func TestEndpointStore(t *testing.T) { }, }, Subsets: []v1.EndpointSubset{ - {Addresses: []v1.EndpointAddress{ - {IP: "127.0.0.1"}, {IP: "10.0.0.1"}, + { + Addresses: []v1.EndpointAddress{ + {IP: "127.0.0.1"}, {IP: "10.0.0.1"}, + }, + NotReadyAddresses: []v1.EndpointAddress{ + {IP: "10.0.0.10"}, + }, + Ports: []v1.EndpointPort{ + {Port: 8080, Name: "http", Protocol: v1.ProtocolTCP}, {Port: 8081, Name: "app", Protocol: v1.ProtocolTCP}, + }, + }, + { + Addresses: []v1.EndpointAddress{ + {IP: "172.22.23.202"}, + }, + Ports: []v1.EndpointPort{ + {Port: 8443, Name: "https", Protocol: v1.ProtocolTCP}, {Port: 9090, Name: "prometheus", Protocol: v1.ProtocolTCP}, + }, }, + { + NotReadyAddresses: []v1.EndpointAddress{ + {IP: "192.168.1.3"}, {IP: "192.168.2.2"}, + }, Ports: []v1.EndpointPort{ - {Port: 8080}, {Port: 8081}, + {Port: 1234, Name: "syslog", Protocol: v1.ProtocolUDP}, {Port: 5678, Name: "syslog-tcp", Protocol: v1.ProtocolTCP}, }, }, - {Addresses: []v1.EndpointAddress{ - {IP: "172.22.23.202"}, + }, + }, + Want: metadata + ` + kube_endpoint_annotations{endpoint="test-endpoint",namespace="default"} 1 + kube_endpoint_address_available{endpoint="test-endpoint",namespace="default"} 6 + kube_endpoint_address_not_ready{endpoint="test-endpoint",namespace="default"} 6 + kube_endpoint_created{endpoint="test-endpoint",namespace="default"} 1.5e+09 + kube_endpoint_info{endpoint="test-endpoint",namespace="default"} 1 + kube_endpoint_labels{endpoint="test-endpoint",namespace="default"} 1 + kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="http",port_protocol="TCP",port_number="8080"} 1 + kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="app",port_protocol="TCP",port_number="8081"} 1 + kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="https",port_protocol="TCP",port_number="8443"} 1 + kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="prometheus",port_protocol="TCP",port_number="9090"} 1 + kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="syslog",port_protocol="UDP",port_number="1234"} 1 + kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="syslog-tcp",port_protocol="TCP",port_number="5678"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="127.0.0.1",ready="true"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="10.0.0.1",ready="true"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="172.22.23.202",ready="true"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="192.168.1.3",ready="false"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="192.168.2.2",ready="false"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="10.0.0.10",ready="false"} 1 + `, + }, + { + Obj: &v1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: "single-port-endpoint", + CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, + Namespace: "default", + Labels: map[string]string{ + "app": "single-foobar", }, + }, + Subsets: []v1.EndpointSubset{ + { + Addresses: []v1.EndpointAddress{ + {IP: "127.0.0.1"}, {IP: "10.0.0.1"}, + }, + NotReadyAddresses: []v1.EndpointAddress{ + {IP: "10.0.0.10"}, + }, Ports: []v1.EndpointPort{ - {Port: 8443}, {Port: 9090}, + {Port: 8080, Protocol: v1.ProtocolTCP}, }, }, - {NotReadyAddresses: []v1.EndpointAddress{ - {IP: "192.168.1.1"}, + }, + }, + Want: metadata + ` + kube_endpoint_annotations{endpoint="single-port-endpoint",namespace="default"} 1 + kube_endpoint_address_available{endpoint="single-port-endpoint",namespace="default"} 2 + kube_endpoint_address_not_ready{endpoint="single-port-endpoint",namespace="default"} 1 + kube_endpoint_created{endpoint="single-port-endpoint",namespace="default"} 1.5e+09 + kube_endpoint_info{endpoint="single-port-endpoint",namespace="default"} 1 + kube_endpoint_labels{endpoint="single-port-endpoint",namespace="default"} 1 + kube_endpoint_ports{endpoint="single-port-endpoint",namespace="default",port_name="",port_number="8080",port_protocol="TCP"} 1 + kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",ip="127.0.0.1",ready="true"} 1 + kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",ip="10.0.0.1",ready="true"} 1 + kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",ip="10.0.0.10",ready="false"} 1 + `, + }, + } + for i, c := range cases { + c.Func = generator.ComposeMetricGenFuncs(endpointMetricFamilies(nil, nil)) + c.Headers = generator.ExtractMetricFamilyHeaders(endpointMetricFamilies(nil, nil)) + if err := c.run(); err != nil { + t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) + } + } +} + +func TestEndpointStoreWithLabels(t *testing.T) { + // Fixed metadata on type and help text. We prepend this to every expected + // output so we only have to modify a single place when doing adjustments. + const metadata = ` + # HELP kube_endpoint_address_available (Deprecated since v2.6.0) Number of addresses available in endpoint. + # TYPE kube_endpoint_address_available gauge + # HELP kube_endpoint_address_not_ready (Deprecated since v2.6.0) Number of addresses not ready in endpoint + # TYPE kube_endpoint_address_not_ready gauge + # HELP kube_endpoint_annotations Kubernetes annotations converted to Prometheus labels. + # TYPE kube_endpoint_annotations gauge + # HELP kube_endpoint_created [STABLE] Unix creation timestamp + # TYPE kube_endpoint_created gauge + # HELP kube_endpoint_info [STABLE] Information about endpoint. + # TYPE kube_endpoint_info gauge + # HELP kube_endpoint_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # TYPE kube_endpoint_labels gauge + # HELP kube_endpoint_ports [STABLE] Information about the Endpoint ports. + # TYPE kube_endpoint_ports gauge + # HELP kube_endpoint_address [STABLE] Information about Endpoint available and non available addresses. + # TYPE kube_endpoint_address gauge + ` + cases := []generateMetricsTestCase{ + { + Obj: &v1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-endpoint", + CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, + Namespace: "default", + Annotations: map[string]string{ + "app": "foobar", }, + Labels: map[string]string{ + "app": "foobar", + }, + }, + Subsets: []v1.EndpointSubset{ + { + Addresses: []v1.EndpointAddress{ + {IP: "127.0.0.1"}, {IP: "10.0.0.1"}, + }, + NotReadyAddresses: []v1.EndpointAddress{ + {IP: "10.0.0.10"}, + }, Ports: []v1.EndpointPort{ - {Port: 1234}, {Port: 5678}, + {Port: 8080, Name: "http", Protocol: v1.ProtocolTCP}, {Port: 8081, Name: "app", Protocol: v1.ProtocolTCP}, }, }, - {NotReadyAddresses: []v1.EndpointAddress{ - {IP: "192.168.1.3"}, {IP: "192.168.2.2"}, + { + Addresses: []v1.EndpointAddress{ + {IP: "172.22.23.202"}, + }, + Ports: []v1.EndpointPort{ + {Port: 8443, Name: "https", Protocol: v1.ProtocolTCP}, {Port: 9090, Name: "prometheus", Protocol: v1.ProtocolTCP}, + }, }, + { + NotReadyAddresses: []v1.EndpointAddress{ + {IP: "192.168.1.3"}, {IP: "192.168.2.2"}, + }, Ports: []v1.EndpointPort{ - {Port: 1234}, {Port: 5678}, + {Port: 1234, Name: "syslog", Protocol: v1.ProtocolUDP}, {Port: 5678, Name: "syslog-tcp", Protocol: v1.ProtocolTCP}, }, }, }, @@ -86,15 +224,74 @@ func TestEndpointStore(t *testing.T) { Want: metadata + ` kube_endpoint_address_available{endpoint="test-endpoint",namespace="default"} 6 kube_endpoint_address_not_ready{endpoint="test-endpoint",namespace="default"} 6 + kube_endpoint_annotations{endpoint="test-endpoint",annotation_app="foobar",namespace="default"} 1 kube_endpoint_created{endpoint="test-endpoint",namespace="default"} 1.5e+09 kube_endpoint_info{endpoint="test-endpoint",namespace="default"} 1 kube_endpoint_labels{endpoint="test-endpoint",label_app="foobar",namespace="default"} 1 + kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="http",port_protocol="TCP",port_number="8080"} 1 + kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="app",port_protocol="TCP",port_number="8081"} 1 + kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="https",port_protocol="TCP",port_number="8443"} 1 + kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="prometheus",port_protocol="TCP",port_number="9090"} 1 + kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="syslog",port_protocol="UDP",port_number="1234"} 1 + kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="syslog-tcp",port_protocol="TCP",port_number="5678"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="127.0.0.1",ready="true"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="10.0.0.1",ready="true"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="172.22.23.202",ready="true"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="192.168.1.3",ready="false"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="192.168.2.2",ready="false"} 1 + kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="10.0.0.10",ready="false"} 1 + `, + }, + { + Obj: &v1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: "single-port-endpoint", + CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, + Namespace: "default", + Annotations: map[string]string{ + "app": "single-foobar", + }, + Labels: map[string]string{ + "app": "single-foobar", + }, + }, + Subsets: []v1.EndpointSubset{ + { + Addresses: []v1.EndpointAddress{ + {IP: "127.0.0.1"}, {IP: "10.0.0.1"}, + }, + NotReadyAddresses: []v1.EndpointAddress{ + {IP: "10.0.0.10"}, + }, + Ports: []v1.EndpointPort{ + {Port: 8080, Protocol: v1.ProtocolTCP}, + }, + }, + }, + }, + Want: metadata + ` + kube_endpoint_annotations{endpoint="single-port-endpoint",annotation_app="single-foobar",namespace="default"} 1 + kube_endpoint_address_available{endpoint="single-port-endpoint",namespace="default"} 2 + kube_endpoint_address_not_ready{endpoint="single-port-endpoint",namespace="default"} 1 + kube_endpoint_created{endpoint="single-port-endpoint",namespace="default"} 1.5e+09 + kube_endpoint_info{endpoint="single-port-endpoint",namespace="default"} 1 + kube_endpoint_labels{endpoint="single-port-endpoint",label_app="single-foobar",namespace="default"} 1 + kube_endpoint_ports{endpoint="single-port-endpoint",namespace="default",port_name="",port_number="8080",port_protocol="TCP"} 1 + kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",ip="127.0.0.1",ready="true"} 1 + kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",ip="10.0.0.1",ready="true"} 1 + kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",ip="10.0.0.10",ready="false"} 1 `, }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(endpointMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(endpointMetricFamilies) + allowAnnotations := []string{ + "app", + } + allowLabels := []string{ + "app", + } + c.Func = generator.ComposeMetricGenFuncs(endpointMetricFamilies(allowAnnotations, allowLabels)) + c.Headers = generator.ExtractMetricFamilyHeaders(endpointMetricFamilies(allowAnnotations, allowLabels)) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/endpointslice.go b/internal/store/endpointslice.go new file mode 100644 index 0000000000..0b01284468 --- /dev/null +++ b/internal/store/endpointslice.go @@ -0,0 +1,234 @@ +/* +Copyright 2022 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "context" + "strconv" + + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" + + discoveryv1 "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" +) + +var ( + descEndpointSliceAnnotationsName = "kube_endpointslice_annotations" + descEndpointSliceAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." + descEndpointSliceLabelsName = "kube_endpointslice_labels" + descEndpointSliceLabelsHelp = "Kubernetes labels converted to Prometheus labels." + descEndpointSliceLabelsDefaultLabels = []string{"endpointslice"} +) + +func endpointSliceMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_endpointslice_info", + "Information about endpointslice.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapEndpointSliceFunc(func(s *discoveryv1.EndpointSlice) *metric.Family { + + m := metric.Metric{ + LabelKeys: []string{"addresstype"}, + LabelValues: []string{string(s.AddressType)}, + Value: 1, + } + return &metric.Family{Metrics: []*metric.Metric{&m}} + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_endpointslice_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapEndpointSliceFunc(func(s *discoveryv1.EndpointSlice) *metric.Family { + ms := []*metric.Metric{} + if !s.CreationTimestamp.IsZero() { + ms = append(ms, &metric.Metric{ + Value: float64(s.CreationTimestamp.Unix()), + }) + } + return &metric.Family{ + Metrics: ms, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_endpointslice_endpoints", + "Endpoints attached to the endpointslice.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapEndpointSliceFunc(func(e *discoveryv1.EndpointSlice) *metric.Family { + m := []*metric.Metric{} + for _, ep := range e.Endpoints { + var ( + labelKeys, + labelValues []string + ) + + if ep.Conditions.Ready != nil { + labelKeys = append(labelKeys, "ready") + labelValues = append(labelValues, strconv.FormatBool(*ep.Conditions.Ready)) + } + if ep.Conditions.Serving != nil { + labelKeys = append(labelKeys, "serving") + labelValues = append(labelValues, strconv.FormatBool(*ep.Conditions.Serving)) + } + if ep.Conditions.Terminating != nil { + labelKeys = append(labelKeys, "terminating") + labelValues = append(labelValues, strconv.FormatBool(*ep.Conditions.Terminating)) + } + + if ep.Hostname != nil { + labelKeys = append(labelKeys, "hostname") + labelValues = append(labelValues, *ep.Hostname) + } + + if ep.TargetRef != nil { + if ep.TargetRef.Kind != "" { + labelKeys = append(labelKeys, "targetref_kind") + labelValues = append(labelValues, ep.TargetRef.Kind) + } + if ep.TargetRef.Name != "" { + labelKeys = append(labelKeys, "targetref_name") + labelValues = append(labelValues, ep.TargetRef.Name) + } + if ep.TargetRef.Namespace != "" { + labelKeys = append(labelKeys, "targetref_namespace") + labelValues = append(labelValues, ep.TargetRef.Namespace) + } + } + + if ep.NodeName != nil { + labelKeys = append(labelKeys, "endpoint_nodename") + labelValues = append(labelValues, *ep.NodeName) + } + + if ep.Zone != nil { + labelKeys = append(labelKeys, "endpoint_zone") + labelValues = append(labelValues, *ep.Zone) + } + labelKeys = append(labelKeys, "address") + for _, address := range ep.Addresses { + newlabelValues := make([]string, len(labelValues)) + copy(newlabelValues, labelValues) + m = append(m, &metric.Metric{ + LabelKeys: labelKeys, + LabelValues: append(newlabelValues, address), + Value: 1, + }) + } + } + return &metric.Family{ + Metrics: m, + } + }), + ), + + *generator.NewFamilyGeneratorWithStability( + "kube_endpointslice_ports", + "Ports attached to the endpointslice.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapEndpointSliceFunc(func(e *discoveryv1.EndpointSlice) *metric.Family { + m := []*metric.Metric{} + for _, port := range e.Ports { + m = append(m, &metric.Metric{ + LabelValues: []string{*port.Name, string(*port.Protocol), strconv.FormatInt(int64(*port.Port), 10)}, + LabelKeys: []string{"port_name", "port_protocol", "port_number"}, + Value: 1, + }) + } + return &metric.Family{ + Metrics: m, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descEndpointSliceAnnotationsName, + descEndpointSliceAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapEndpointSliceFunc(func(s *discoveryv1.EndpointSlice) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", s.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descEndpointSliceLabelsName, + descEndpointSliceLabelsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapEndpointSliceFunc(func(s *discoveryv1.EndpointSlice) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", s.Labels, allowLabelsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: labelKeys, + LabelValues: labelValues, + Value: 1, + }, + }, + } + }), + ), + } +} + +func wrapEndpointSliceFunc(f func(*discoveryv1.EndpointSlice) *metric.Family) func(interface{}) *metric.Family { + return func(obj interface{}) *metric.Family { + endpointSlice := obj.(*discoveryv1.EndpointSlice) + + metricFamily := f(endpointSlice) + + for _, m := range metricFamily.Metrics { + m.LabelKeys, m.LabelValues = mergeKeyValues(descEndpointSliceLabelsDefaultLabels, []string{endpointSlice.Name}, m.LabelKeys, m.LabelValues) + } + + return metricFamily + } +} + +func createEndpointSliceListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { + return &cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + return kubeClient.DiscoveryV1().EndpointSlices(ns).List(context.TODO(), opts) + }, + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + return kubeClient.DiscoveryV1().EndpointSlices(ns).Watch(context.TODO(), opts) + }, + } +} diff --git a/internal/store/endpointslice_test.go b/internal/store/endpointslice_test.go new file mode 100644 index 0000000000..cca8638cd9 --- /dev/null +++ b/internal/store/endpointslice_test.go @@ -0,0 +1,161 @@ +/* +Copyright 2022 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +func TestEndpointSliceStore(t *testing.T) { + startTime := 1501569018 + metav1StartTime := metav1.Unix(int64(startTime), 0) + portname := "http" + portnumber := int32(80) + portprotocol := corev1.Protocol("TCP") + nodename := "node" + hostname := "host" + zone := "west" + ready := true + terminating := false + addresses := []string{"10.0.0.1", "192.168.1.10"} + + cases := []generateMetricsTestCase{ + { + Obj: &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test_endpointslice-info", + }, + AddressType: "IPv4", + }, + Want: ` + # HELP kube_endpointslice_info Information about endpointslice. + # TYPE kube_endpointslice_info gauge + kube_endpointslice_info{endpointslice="test_endpointslice-info",addresstype="IPv4"} 1 + `, + MetricNames: []string{ + "kube_endpointslice_info", + }, + }, + { + Obj: &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test_kube_endpointslice-created", + CreationTimestamp: metav1StartTime, + }, + AddressType: "IPv4", + }, + Want: ` + # HELP kube_endpointslice_created Unix creation timestamp + # TYPE kube_endpointslice_created gauge + kube_endpointslice_created{endpointslice="test_kube_endpointslice-created"} 1.501569018e+09 + `, + MetricNames: []string{ + "kube_endpointslice_created", + }, + }, + { + Obj: &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test_endpointslice-ports", + }, + AddressType: "IPv4", + Ports: []discoveryv1.EndpointPort{ + {Name: &portname, + Port: &portnumber, + Protocol: &portprotocol, + }, + }, + }, + Want: ` + # HELP kube_endpointslice_ports Ports attached to the endpointslice. + # TYPE kube_endpointslice_ports gauge + kube_endpointslice_ports{endpointslice="test_endpointslice-ports",port_name="http",port_protocol="TCP",port_number="80"} 1 + `, + MetricNames: []string{ + "kube_endpointslice_ports", + }, + }, + { + Obj: &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test_endpointslice-endpoints", + }, + AddressType: "IPv4", + Endpoints: []discoveryv1.Endpoint{ + { + NodeName: &nodename, + Conditions: discoveryv1.EndpointConditions{ + Ready: &ready, + Terminating: &terminating, + }, + Hostname: &hostname, + Zone: &zone, + Addresses: addresses, + }, + }, + }, + Want: ` + # HELP kube_endpointslice_endpoints Endpoints attached to the endpointslice. + # TYPE kube_endpointslice_endpoints gauge + kube_endpointslice_endpoints{address="10.0.0.1",endpoint_nodename="node",endpoint_zone="west",endpointslice="test_endpointslice-endpoints",hostname="host",ready="true",terminating="false"} 1 + kube_endpointslice_endpoints{address="192.168.1.10",endpoint_nodename="node",endpoint_zone="west",endpointslice="test_endpointslice-endpoints",hostname="host",ready="true",terminating="false"} 1 + `, + + MetricNames: []string{ + "kube_endpointslice_endpoints", + }, + }, + { + AllowAnnotationsList: []string{ + "foo", + }, + Obj: &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test_endpointslice-labels", + Annotations: map[string]string{ + "foo": "baz", + }, + Labels: map[string]string{ + "foo": "bar", + }, + }, + AddressType: "IPv4", + }, + Want: ` + # HELP kube_endpointslice_annotations Kubernetes annotations converted to Prometheus labels. + # HELP kube_endpointslice_labels Kubernetes labels converted to Prometheus labels. + # TYPE kube_endpointslice_annotations gauge + # TYPE kube_endpointslice_labels gauge + kube_endpointslice_annotations{endpointslice="test_endpointslice-labels",annotation_foo="baz"} 1 + kube_endpointslice_labels{endpointslice="test_endpointslice-labels"} 1 + `, + MetricNames: []string{ + "kube_endpointslice_annotations", "kube_endpointslice_labels", + }, + }, + } + for i, c := range cases { + c.Func = generator.ComposeMetricGenFuncs(endpointSliceMetricFamilies(c.AllowAnnotationsList, nil)) + c.Headers = generator.ExtractMetricFamilyHeaders(endpointSliceMetricFamilies(c.AllowAnnotationsList, nil)) + if err := c.run(); err != nil { + t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) + } + } +} diff --git a/internal/store/horizontalpodautoscaler.go b/internal/store/horizontalpodautoscaler.go new file mode 100644 index 0000000000..241ad2f2b9 --- /dev/null +++ b/internal/store/horizontalpodautoscaler.go @@ -0,0 +1,365 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "context" + + autoscaling "k8s.io/api/autoscaling/v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +type metricTargetType int + +const ( + value metricTargetType = iota + utilization + average +) + +func (m metricTargetType) String() string { + return [...]string{"value", "utilization", "average"}[m] +} + +var ( + descHorizontalPodAutoscalerAnnotationsName = "kube_horizontalpodautoscaler_annotations" + descHorizontalPodAutoscalerAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." + descHorizontalPodAutoscalerLabelsName = "kube_horizontalpodautoscaler_labels" + descHorizontalPodAutoscalerLabelsHelp = "Kubernetes labels converted to Prometheus labels." + descHorizontalPodAutoscalerLabelsDefaultLabels = []string{"namespace", "horizontalpodautoscaler"} + + targetMetricLabels = []string{"metric_name", "metric_target_type"} +) + +func hpaMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_horizontalpodautoscaler_info", + "Information about this autoscaler.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { + labelKeys := []string{"scaletargetref_kind", "scaletargetref_name"} + labelValues := []string{a.Spec.ScaleTargetRef.Kind, a.Spec.ScaleTargetRef.Name} + if a.Spec.ScaleTargetRef.APIVersion != "" { + labelKeys = append([]string{"scaletargetref_api_version"}, labelKeys...) + labelValues = append([]string{a.Spec.ScaleTargetRef.APIVersion}, labelValues...) + } + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: labelKeys, + LabelValues: labelValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_horizontalpodautoscaler_metadata_generation", + "The generation observed by the HorizontalPodAutoscaler controller.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { + return &metric.Family{ + Metrics: []*metric.Metric{ + { + Value: float64(a.ObjectMeta.Generation), + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_horizontalpodautoscaler_spec_max_replicas", + "Upper limit for the number of pods that can be set by the autoscaler; cannot be smaller than MinReplicas.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { + return &metric.Family{ + Metrics: []*metric.Metric{ + { + Value: float64(a.Spec.MaxReplicas), + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_horizontalpodautoscaler_spec_min_replicas", + "Lower limit for the number of pods that can be set by the autoscaler, default 1.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { + return &metric.Family{ + Metrics: []*metric.Metric{ + { + Value: float64(*a.Spec.MinReplicas), + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_horizontalpodautoscaler_spec_target_metric", + "The metric specifications used by this autoscaler when calculating the desired replica count.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { + ms := make([]*metric.Metric, 0, len(a.Spec.Metrics)) + for _, m := range a.Spec.Metrics { + var metricName string + var metricTarget autoscaling.MetricTarget + // The variable maps the type of metric to the corresponding value + metricMap := make(map[metricTargetType]float64) + + switch m.Type { + case autoscaling.ObjectMetricSourceType: + metricName = m.Object.Metric.Name + metricTarget = m.Object.Target + case autoscaling.PodsMetricSourceType: + metricName = m.Pods.Metric.Name + metricTarget = m.Pods.Target + case autoscaling.ResourceMetricSourceType: + metricName = string(m.Resource.Name) + metricTarget = m.Resource.Target + case autoscaling.ContainerResourceMetricSourceType: + metricName = string(m.ContainerResource.Name) + metricTarget = m.ContainerResource.Target + case autoscaling.ExternalMetricSourceType: + metricName = m.External.Metric.Name + metricTarget = m.External.Target + default: + // Skip unsupported metric type + continue + } + + if metricTarget.Value != nil { + metricMap[value] = float64(metricTarget.Value.MilliValue()) / 1000 + } + if metricTarget.AverageValue != nil { + metricMap[average] = float64(metricTarget.AverageValue.MilliValue()) / 1000 + } + if metricTarget.AverageUtilization != nil { + metricMap[utilization] = float64(*metricTarget.AverageUtilization) + } + + for metricTypeIndex, metricValue := range metricMap { + ms = append(ms, &metric.Metric{ + LabelKeys: targetMetricLabels, + LabelValues: []string{metricName, metricTypeIndex.String()}, + Value: metricValue, + }) + } + } + return &metric.Family{Metrics: ms} + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_horizontalpodautoscaler_status_target_metric", + "The current metric status used by this autoscaler when calculating the desired replica count.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { + ms := make([]*metric.Metric, 0, len(a.Status.CurrentMetrics)) + for _, m := range a.Status.CurrentMetrics { + var metricName string + var currentMetric autoscaling.MetricValueStatus + // The variable maps the type of metric to the corresponding value + metricMap := make(map[metricTargetType]float64) + + switch m.Type { + case autoscaling.ObjectMetricSourceType: + metricName = m.Object.Metric.Name + currentMetric = m.Object.Current + case autoscaling.PodsMetricSourceType: + metricName = m.Pods.Metric.Name + currentMetric = m.Pods.Current + case autoscaling.ResourceMetricSourceType: + metricName = string(m.Resource.Name) + currentMetric = m.Resource.Current + case autoscaling.ContainerResourceMetricSourceType: + metricName = string(m.ContainerResource.Name) + currentMetric = m.ContainerResource.Current + case autoscaling.ExternalMetricSourceType: + metricName = m.External.Metric.Name + currentMetric = m.External.Current + default: + // Skip unsupported metric type + continue + } + + if currentMetric.Value != nil { + metricMap[value] = float64(currentMetric.Value.MilliValue()) / 1000 + } + if currentMetric.AverageValue != nil { + metricMap[average] = float64(currentMetric.AverageValue.MilliValue()) / 1000 + } + if currentMetric.AverageUtilization != nil { + metricMap[utilization] = float64(*currentMetric.AverageUtilization) + } + + for metricTypeIndex, metricValue := range metricMap { + ms = append(ms, &metric.Metric{ + LabelKeys: targetMetricLabels, + LabelValues: []string{metricName, metricTypeIndex.String()}, + Value: metricValue, + }) + } + } + return &metric.Family{Metrics: ms} + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_horizontalpodautoscaler_status_current_replicas", + "Current number of replicas of pods managed by this autoscaler.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { + return &metric.Family{ + Metrics: []*metric.Metric{ + { + Value: float64(a.Status.CurrentReplicas), + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_horizontalpodautoscaler_status_desired_replicas", + "Desired number of replicas of pods managed by this autoscaler.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { + return &metric.Family{ + Metrics: []*metric.Metric{ + { + Value: float64(a.Status.DesiredReplicas), + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descHorizontalPodAutoscalerAnnotationsName, + descHorizontalPodAutoscalerAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", a.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descHorizontalPodAutoscalerLabelsName, + descHorizontalPodAutoscalerLabelsHelp, + metric.Gauge, + basemetrics.STABLE, + "", + wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", a.Labels, allowLabelsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: labelKeys, + LabelValues: labelValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_horizontalpodautoscaler_status_condition", + "The condition of this autoscaler.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { + ms := make([]*metric.Metric, 0, len(a.Status.Conditions)*len(conditionStatuses)) + + for _, c := range a.Status.Conditions { + metrics := addConditionMetrics(c.Status) + + for _, m := range metrics { + metric := m + metric.LabelKeys = []string{"condition", "status"} + metric.LabelValues = append([]string{string(c.Type)}, metric.LabelValues...) + ms = append(ms, metric) + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ), + } +} + +func wrapHPAFunc(f func(*autoscaling.HorizontalPodAutoscaler) *metric.Family) func(interface{}) *metric.Family { + return func(obj interface{}) *metric.Family { + hpa := obj.(*autoscaling.HorizontalPodAutoscaler) + + metricFamily := f(hpa) + + for _, m := range metricFamily.Metrics { + m.LabelKeys, m.LabelValues = mergeKeyValues(descHorizontalPodAutoscalerLabelsDefaultLabels, []string{hpa.Namespace, hpa.Name}, m.LabelKeys, m.LabelValues) + } + + return metricFamily + } +} + +func createHPAListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { + return &cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + opts.FieldSelector = fieldSelector + return kubeClient.AutoscalingV2().HorizontalPodAutoscalers(ns).List(context.TODO(), opts) + }, + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + opts.FieldSelector = fieldSelector + return kubeClient.AutoscalingV2().HorizontalPodAutoscalers(ns).Watch(context.TODO(), opts) + }, + } +} diff --git a/internal/store/horizontalpodautoscaler_test.go b/internal/store/horizontalpodautoscaler_test.go new file mode 100644 index 0000000000..85c7f51b2b --- /dev/null +++ b/internal/store/horizontalpodautoscaler_test.go @@ -0,0 +1,445 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "testing" + + autoscaling "k8s.io/api/autoscaling/v2" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +var ( + hpa1MinReplicas int32 = 2 +) + +func TestHPAStore(t *testing.T) { + // Fixed metadata on type and help text. We prepend this to every expected + // output so we only have to modify a single place when doing adjustments. + const metadata = ` + # HELP kube_horizontalpodautoscaler_info Information about this autoscaler. + # HELP kube_horizontalpodautoscaler_annotations Kubernetes annotations converted to Prometheus labels. + # HELP kube_horizontalpodautoscaler_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # HELP kube_horizontalpodautoscaler_metadata_generation [STABLE] The generation observed by the HorizontalPodAutoscaler controller. + # HELP kube_horizontalpodautoscaler_spec_max_replicas [STABLE] Upper limit for the number of pods that can be set by the autoscaler; cannot be smaller than MinReplicas. + # HELP kube_horizontalpodautoscaler_spec_min_replicas [STABLE] Lower limit for the number of pods that can be set by the autoscaler, default 1. + # HELP kube_horizontalpodautoscaler_spec_target_metric The metric specifications used by this autoscaler when calculating the desired replica count. + # HELP kube_horizontalpodautoscaler_status_target_metric The current metric status used by this autoscaler when calculating the desired replica count. + # HELP kube_horizontalpodautoscaler_status_condition [STABLE] The condition of this autoscaler. + # HELP kube_horizontalpodautoscaler_status_current_replicas [STABLE] Current number of replicas of pods managed by this autoscaler. + # HELP kube_horizontalpodautoscaler_status_desired_replicas [STABLE] Desired number of replicas of pods managed by this autoscaler. + # TYPE kube_horizontalpodautoscaler_info gauge + # TYPE kube_horizontalpodautoscaler_annotations gauge + # TYPE kube_horizontalpodautoscaler_labels gauge + # TYPE kube_horizontalpodautoscaler_metadata_generation gauge + # TYPE kube_horizontalpodautoscaler_spec_max_replicas gauge + # TYPE kube_horizontalpodautoscaler_spec_min_replicas gauge + # TYPE kube_horizontalpodautoscaler_spec_target_metric gauge + # TYPE kube_horizontalpodautoscaler_status_target_metric gauge + # TYPE kube_horizontalpodautoscaler_status_condition gauge + # TYPE kube_horizontalpodautoscaler_status_current_replicas gauge + # TYPE kube_horizontalpodautoscaler_status_desired_replicas gauge + ` + cases := []generateMetricsTestCase{ + { + // Verify populating base metric. + Obj: &autoscaling.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 2, + Name: "hpa1", + Namespace: "ns1", + Labels: map[string]string{ + "app": "foobar", + }, + }, + Spec: autoscaling.HorizontalPodAutoscalerSpec{ + MaxReplicas: 4, + MinReplicas: &hpa1MinReplicas, + Metrics: []autoscaling.MetricSpec{ + { + Type: autoscaling.ObjectMetricSourceType, + Object: &autoscaling.ObjectMetricSource{ + Metric: autoscaling.MetricIdentifier{ + Name: "hits", + }, + Target: autoscaling.MetricTarget{ + Value: resourcePtr(resource.MustParse("10")), + AverageValue: resourcePtr(resource.MustParse("12")), + }, + }, + }, + { + Type: autoscaling.ObjectMetricSourceType, + Object: &autoscaling.ObjectMetricSource{ + Metric: autoscaling.MetricIdentifier{ + Name: "connections", + }, + Target: autoscaling.MetricTarget{ + Value: resourcePtr(resource.MustParse("0.5")), + AverageValue: resourcePtr(resource.MustParse("0.7")), + }, + }, + }, + { + Type: autoscaling.PodsMetricSourceType, + Pods: &autoscaling.PodsMetricSource{ + Metric: autoscaling.MetricIdentifier{ + Name: "transactions_processed", + }, + Target: autoscaling.MetricTarget{ + AverageValue: resourcePtr(resource.MustParse("33")), + }, + }, + }, + { + Type: autoscaling.ResourceMetricSourceType, + Resource: &autoscaling.ResourceMetricSource{ + Name: "cpu", + Target: autoscaling.MetricTarget{ + AverageUtilization: int32ptr(80), + }, + }, + }, + { + Type: autoscaling.ResourceMetricSourceType, + Resource: &autoscaling.ResourceMetricSource{ + Name: "memory", + Target: autoscaling.MetricTarget{ + AverageValue: resourcePtr(resource.MustParse("800Ki")), + AverageUtilization: int32ptr(80), + }, + }, + }, + { + Type: autoscaling.ContainerResourceMetricSourceType, + ContainerResource: &autoscaling.ContainerResourceMetricSource{ + Name: "cpu", + Container: "container1", + Target: autoscaling.MetricTarget{ + AverageUtilization: int32ptr(80), + }, + }, + }, + // No targets, this metric should be ignored + { + Type: autoscaling.ResourceMetricSourceType, + Resource: &autoscaling.ResourceMetricSource{ + Name: "disk", + }, + }, + { + Type: autoscaling.ExternalMetricSourceType, + External: &autoscaling.ExternalMetricSource{ + Metric: autoscaling.MetricIdentifier{ + Name: "sqs_jobs", + }, + Target: autoscaling.MetricTarget{ + Value: resourcePtr(resource.MustParse("30")), + }, + }, + }, + { + Type: autoscaling.ExternalMetricSourceType, + External: &autoscaling.ExternalMetricSource{ + Metric: autoscaling.MetricIdentifier{ + Name: "events", + }, + Target: autoscaling.MetricTarget{ + AverageValue: resourcePtr(resource.MustParse("30")), + }, + }, + }, + }, + ScaleTargetRef: autoscaling.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "deployment1", + }, + }, + Status: autoscaling.HorizontalPodAutoscalerStatus{ + CurrentReplicas: 2, + DesiredReplicas: 2, + Conditions: []autoscaling.HorizontalPodAutoscalerCondition{ + { + Type: autoscaling.AbleToScale, + Status: v1.ConditionTrue, + Reason: "reason", + }, + }, + CurrentMetrics: []autoscaling.MetricStatus{ + { + Type: "Resource", + Resource: &autoscaling.ResourceMetricStatus{ + Name: "cpu", + Current: autoscaling.MetricValueStatus{ + AverageValue: resourcePtr(resource.MustParse("7m")), + AverageUtilization: int32ptr(80), + }, + }, + }, + { + Type: "Resource", + Resource: &autoscaling.ResourceMetricStatus{ + Name: "memory", + Current: autoscaling.MetricValueStatus{ + AverageValue: resourcePtr(resource.MustParse("26335914666m")), + AverageUtilization: int32ptr(80), + }, + }, + }, + }, + }, + }, + Want: metadata + ` + kube_horizontalpodautoscaler_info{horizontalpodautoscaler="hpa1",namespace="ns1",scaletargetref_api_version="apps/v1",scaletargetref_kind="Deployment",scaletargetref_name="deployment1"} 1 + kube_horizontalpodautoscaler_annotations{horizontalpodautoscaler="hpa1",namespace="ns1"} 1 + kube_horizontalpodautoscaler_labels{horizontalpodautoscaler="hpa1",namespace="ns1"} 1 + kube_horizontalpodautoscaler_metadata_generation{horizontalpodautoscaler="hpa1",namespace="ns1"} 2 + kube_horizontalpodautoscaler_spec_max_replicas{horizontalpodautoscaler="hpa1",namespace="ns1"} 4 + kube_horizontalpodautoscaler_spec_min_replicas{horizontalpodautoscaler="hpa1",namespace="ns1"} 2 + kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="cpu",metric_target_type="utilization",namespace="ns1"} 80 + kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="events",metric_target_type="average",namespace="ns1"} 30 + kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="hits",metric_target_type="average",namespace="ns1"} 12 + kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="hits",metric_target_type="value",namespace="ns1"} 10 + kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="connections",metric_target_type="average",namespace="ns1"} 0.7 + kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="connections",metric_target_type="value",namespace="ns1"} 0.5 + kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="cpu",metric_target_type="utilization",namespace="ns1"} 80 + kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="memory",metric_target_type="average",namespace="ns1"} 819200 + kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="memory",metric_target_type="utilization",namespace="ns1"} 80 + kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="sqs_jobs",metric_target_type="value",namespace="ns1"} 30 + kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="transactions_processed",metric_target_type="average",namespace="ns1"} 33 + kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa1",metric_name="cpu",metric_target_type="average",namespace="ns1"} 0.007 + kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa1",metric_name="cpu",metric_target_type="utilization",namespace="ns1"} 80 + kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa1",metric_name="memory",metric_target_type="average",namespace="ns1"} 2.6335914666e+07 + kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa1",metric_name="memory",metric_target_type="utilization",namespace="ns1"} 80 + kube_horizontalpodautoscaler_status_condition{condition="AbleToScale",horizontalpodautoscaler="hpa1",namespace="ns1",status="false"} 0 + kube_horizontalpodautoscaler_status_condition{condition="AbleToScale",horizontalpodautoscaler="hpa1",namespace="ns1",status="true"} 1 + kube_horizontalpodautoscaler_status_condition{condition="AbleToScale",horizontalpodautoscaler="hpa1",namespace="ns1",status="unknown"} 0 + kube_horizontalpodautoscaler_status_current_replicas{horizontalpodautoscaler="hpa1",namespace="ns1"} 2 + kube_horizontalpodautoscaler_status_desired_replicas{horizontalpodautoscaler="hpa1",namespace="ns1"} 2 + `, + MetricNames: []string{ + "kube_horizontalpodautoscaler_info", + "kube_horizontalpodautoscaler_metadata_generation", + "kube_horizontalpodautoscaler_spec_max_replicas", + "kube_horizontalpodautoscaler_spec_min_replicas", + "kube_horizontalpodautoscaler_spec_target_metric", + "kube_horizontalpodautoscaler_status_target_metric", + "kube_horizontalpodautoscaler_status_current_replicas", + "kube_horizontalpodautoscaler_status_desired_replicas", + "kube_horizontalpodautoscaler_status_condition", + "kube_horizontalpodautoscaler_annotations", + "kube_horizontalpodautoscaler_labels", + }, + }, + { + // Verify populating base metric. + AllowAnnotationsList: []string{ + "app.k8s.io/owner", + }, + Obj: &autoscaling.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 2, + Name: "hpa2", + Namespace: "ns1", + Labels: map[string]string{ + "app": "foobar", + }, + Annotations: map[string]string{ + "app": "mysql-server", + "app.k8s.io/owner": "@foo", + }, + }, + Spec: autoscaling.HorizontalPodAutoscalerSpec{ + MaxReplicas: 4, + MinReplicas: &hpa1MinReplicas, + Metrics: []autoscaling.MetricSpec{ + { + Type: autoscaling.ResourceMetricSourceType, + Resource: &autoscaling.ResourceMetricSource{ + Name: "memory", + Target: autoscaling.MetricTarget{ + AverageUtilization: int32ptr(75), + }, + }, + }, + { + Type: autoscaling.ResourceMetricSourceType, + Resource: &autoscaling.ResourceMetricSource{ + Name: "cpu", + Target: autoscaling.MetricTarget{ + AverageUtilization: int32ptr(80), + }, + }, + }, + { + Type: autoscaling.ExternalMetricSourceType, + External: &autoscaling.ExternalMetricSource{ + Metric: autoscaling.MetricIdentifier{ + Name: "traefik_backend_requests_per_second", + }, + Target: autoscaling.MetricTarget{ + Value: resourcePtr(resource.MustParse("100")), + }, + }, + }, + { + Type: autoscaling.ExternalMetricSourceType, + External: &autoscaling.ExternalMetricSource{ + Metric: autoscaling.MetricIdentifier{ + Name: "traefik_backend_errors_per_second", + }, + Target: autoscaling.MetricTarget{ + Value: resourcePtr(resource.MustParse("100")), + }, + }, + }, + }, + ScaleTargetRef: autoscaling.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "deployment1", + }, + }, + Status: autoscaling.HorizontalPodAutoscalerStatus{ + CurrentReplicas: 2, + DesiredReplicas: 2, + Conditions: []autoscaling.HorizontalPodAutoscalerCondition{ + { + Type: autoscaling.AbleToScale, + Status: v1.ConditionTrue, + Reason: "reason", + }, + }, + CurrentMetrics: []autoscaling.MetricStatus{ + { + Type: "Resource", + Resource: &autoscaling.ResourceMetricStatus{ + Name: "memory", + Current: autoscaling.MetricValueStatus{ + AverageValue: resourcePtr(resource.MustParse("847775744")), + AverageUtilization: int32ptr(28), + }, + }, + }, + { + Type: "Resource", + Resource: &autoscaling.ResourceMetricStatus{ + Name: "cpu", + Current: autoscaling.MetricValueStatus{ + AverageValue: resourcePtr(resource.MustParse("62m")), + AverageUtilization: int32ptr(6), + }, + }, + }, + { + Type: "ContainerResource", + ContainerResource: &autoscaling.ContainerResourceMetricStatus{ + Name: "cpu", + Container: "container1", + Current: autoscaling.MetricValueStatus{ + AverageValue: resourcePtr(resource.MustParse("80m")), + AverageUtilization: int32ptr(10), + }, + }, + }, + { + Type: "External", + External: &autoscaling.ExternalMetricStatus{ + Metric: autoscaling.MetricIdentifier{ + Name: "traefik_backend_requests_per_second", + }, + Current: autoscaling.MetricValueStatus{ + Value: resourcePtr(resource.MustParse("0")), + AverageValue: resourcePtr(resource.MustParse("2900m")), + }, + }, + }, + { + Type: "External", + External: &autoscaling.ExternalMetricStatus{ + Metric: autoscaling.MetricIdentifier{ + Name: "traefik_backend_errors_per_second", + }, + Current: autoscaling.MetricValueStatus{ + Value: resourcePtr(resource.MustParse("0")), + }, + }, + }, + }, + }, + }, + Want: metadata + ` + kube_horizontalpodautoscaler_info{horizontalpodautoscaler="hpa2",namespace="ns1",scaletargetref_kind="Deployment",scaletargetref_name="deployment1"} 1 + kube_horizontalpodautoscaler_annotations{annotation_app_k8s_io_owner="@foo",horizontalpodautoscaler="hpa2",namespace="ns1"} 1 + kube_horizontalpodautoscaler_labels{horizontalpodautoscaler="hpa2",namespace="ns1"} 1 + kube_horizontalpodautoscaler_metadata_generation{horizontalpodautoscaler="hpa2",namespace="ns1"} 2 + kube_horizontalpodautoscaler_spec_max_replicas{horizontalpodautoscaler="hpa2",namespace="ns1"} 4 + kube_horizontalpodautoscaler_spec_min_replicas{horizontalpodautoscaler="hpa2",namespace="ns1"} 2 + kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa2",metric_name="cpu",metric_target_type="utilization",namespace="ns1"} 80 + kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa2",metric_name="memory",metric_target_type="utilization",namespace="ns1"} 75 + kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa2",metric_name="traefik_backend_errors_per_second",metric_target_type="value",namespace="ns1"} 100 + kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa2",metric_name="traefik_backend_requests_per_second",metric_target_type="value",namespace="ns1"} 100 + kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="memory",metric_target_type="average",namespace="ns1"} 8.47775744e+08 + kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="memory",metric_target_type="utilization",namespace="ns1"} 28 + kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="cpu",metric_target_type="average",namespace="ns1"} 0.062 + kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="cpu",metric_target_type="utilization",namespace="ns1"} 6 + kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="cpu",metric_target_type="average",namespace="ns1"} 0.08 + kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="cpu",metric_target_type="utilization",namespace="ns1"} 10 + kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="traefik_backend_requests_per_second",metric_target_type="value",namespace="ns1"} 0 + kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="traefik_backend_requests_per_second",metric_target_type="average",namespace="ns1"} 2.9 + kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="traefik_backend_errors_per_second",metric_target_type="value",namespace="ns1"} 0 + kube_horizontalpodautoscaler_status_condition{condition="AbleToScale",horizontalpodautoscaler="hpa2",namespace="ns1",status="false"} 0 + kube_horizontalpodautoscaler_status_condition{condition="AbleToScale",horizontalpodautoscaler="hpa2",namespace="ns1",status="true"} 1 + kube_horizontalpodautoscaler_status_condition{condition="AbleToScale",horizontalpodautoscaler="hpa2",namespace="ns1",status="unknown"} 0 + kube_horizontalpodautoscaler_status_current_replicas{horizontalpodautoscaler="hpa2",namespace="ns1"} 2 + kube_horizontalpodautoscaler_status_desired_replicas{horizontalpodautoscaler="hpa2",namespace="ns1"} 2 + `, + MetricNames: []string{ + "kube_horizontalpodautoscaler_info", + "kube_horizontalpodautoscaler_metadata_generation", + "kube_horizontalpodautoscaler_spec_max_replicas", + "kube_horizontalpodautoscaler_spec_min_replicas", + "kube_horizontalpodautoscaler_spec_target_metric", + "kube_horizontalpodautoscaler_status_target_metric", + "kube_horizontalpodautoscaler_status_current_replicas", + "kube_horizontalpodautoscaler_status_desired_replicas", + "kube_horizontalpodautoscaler_status_condition", + "kube_horizontalpodautoscaler_annotation", + "kube_horizontalpodautoscaler_labels", + }, + }, + } + for i, c := range cases { + c.Func = generator.ComposeMetricGenFuncs(hpaMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) + c.Headers = generator.ExtractMetricFamilyHeaders(hpaMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) + if err := c.run(); err != nil { + t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) + } + } +} + +func int32ptr(value int32) *int32 { + return &value +} + +func resourcePtr(quantity resource.Quantity) *resource.Quantity { + return &quantity +} diff --git a/internal/store/hpa.go b/internal/store/hpa.go deleted file mode 100644 index ca7f0ded58..0000000000 --- a/internal/store/hpa.go +++ /dev/null @@ -1,251 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package store - -import ( - autoscaling "k8s.io/api/autoscaling/v2beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/watch" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - - "k8s.io/kube-state-metrics/pkg/metric" -) - -type MetricTargetType int - -const ( - Value MetricTargetType = iota - Utilization - Average - - MetricTargetTypeCount // Used as a length argument to arrays -) - -func (m MetricTargetType) String() string { - return [...]string{"value", "utilization", "average"}[m] -} - -var ( - descHorizontalPodAutoscalerLabelsName = "kube_hpa_labels" - descHorizontalPodAutoscalerLabelsHelp = "Kubernetes labels converted to Prometheus labels." - descHorizontalPodAutoscalerLabelsDefaultLabels = []string{"namespace", "hpa"} - - targetMetricLabels = []string{"metric_name", "metric_target_type"} - - hpaMetricFamilies = []metric.FamilyGenerator{ - { - Name: "kube_hpa_metadata_generation", - Type: metric.Gauge, - Help: "The generation observed by the HorizontalPodAutoscaler controller.", - GenerateFunc: wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { - return &metric.Family{ - Metrics: []*metric.Metric{ - { - Value: float64(a.ObjectMeta.Generation), - }, - }, - } - }), - }, - { - Name: "kube_hpa_spec_max_replicas", - Type: metric.Gauge, - Help: "Upper limit for the number of pods that can be set by the autoscaler; cannot be smaller than MinReplicas.", - GenerateFunc: wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { - return &metric.Family{ - Metrics: []*metric.Metric{ - { - Value: float64(a.Spec.MaxReplicas), - }, - }, - } - }), - }, - { - Name: "kube_hpa_spec_min_replicas", - Type: metric.Gauge, - Help: "Lower limit for the number of pods that can be set by the autoscaler, default 1.", - GenerateFunc: wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { - return &metric.Family{ - Metrics: []*metric.Metric{ - { - Value: float64(*a.Spec.MinReplicas), - }, - }, - } - }), - }, - { - Name: "kube_hpa_spec_target_metric", - Type: metric.Gauge, - Help: "The metric specifications used by this autoscaler when calculating the desired replica count.", - GenerateFunc: wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { - ms := make([]*metric.Metric, 0, len(a.Spec.Metrics)) - for _, m := range a.Spec.Metrics { - var metricName string - - var v [MetricTargetTypeCount]int64 - var ok [MetricTargetTypeCount]bool - - switch m.Type { - case autoscaling.ObjectMetricSourceType: - metricName = m.Object.MetricName - - v[Value], ok[Value] = m.Object.TargetValue.AsInt64() - if m.Object.AverageValue != nil { - v[Average], ok[Average] = m.Object.AverageValue.AsInt64() - } - case autoscaling.PodsMetricSourceType: - metricName = m.Pods.MetricName - - v[Average], ok[Average] = m.Pods.TargetAverageValue.AsInt64() - case autoscaling.ResourceMetricSourceType: - metricName = string(m.Resource.Name) - - if ok[Utilization] = (m.Resource.TargetAverageUtilization != nil); ok[Utilization] { - v[Utilization] = int64(*m.Resource.TargetAverageUtilization) - } - - if m.Resource.TargetAverageValue != nil { - v[Average], ok[Average] = m.Resource.TargetAverageValue.AsInt64() - } - case autoscaling.ExternalMetricSourceType: - metricName = m.External.MetricName - - // The TargetValue and TargetAverageValue are mutually exclusive - if m.External.TargetValue != nil { - v[Value], ok[Value] = m.External.TargetValue.AsInt64() - } - if m.External.TargetAverageValue != nil { - v[Average], ok[Average] = m.External.TargetAverageValue.AsInt64() - } - default: - // Skip unsupported metric type - continue - } - - for i := range ok { - if ok[i] { - ms = append(ms, &metric.Metric{ - LabelKeys: targetMetricLabels, - LabelValues: []string{metricName, MetricTargetType(i).String()}, - Value: float64(v[i]), - }) - } - } - } - return &metric.Family{Metrics: ms} - }), - }, - { - Name: "kube_hpa_status_current_replicas", - Type: metric.Gauge, - Help: "Current number of replicas of pods managed by this autoscaler.", - GenerateFunc: wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { - return &metric.Family{ - Metrics: []*metric.Metric{ - { - Value: float64(a.Status.CurrentReplicas), - }, - }, - } - }), - }, - { - Name: "kube_hpa_status_desired_replicas", - Type: metric.Gauge, - Help: "Desired number of replicas of pods managed by this autoscaler.", - GenerateFunc: wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { - return &metric.Family{ - Metrics: []*metric.Metric{ - { - Value: float64(a.Status.DesiredReplicas), - }, - }, - } - }), - }, - { - Name: descHorizontalPodAutoscalerLabelsName, - Type: metric.Gauge, - Help: descHorizontalPodAutoscalerLabelsHelp, - GenerateFunc: wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(a.Labels) - return &metric.Family{ - Metrics: []*metric.Metric{ - { - LabelKeys: labelKeys, - LabelValues: labelValues, - Value: 1, - }, - }, - } - }), - }, - { - Name: "kube_hpa_status_condition", - Type: metric.Gauge, - Help: "The condition of this autoscaler.", - GenerateFunc: wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { - ms := make([]*metric.Metric, 0, len(a.Status.Conditions)*len(conditionStatuses)) - - for _, c := range a.Status.Conditions { - metrics := addConditionMetrics(c.Status) - - for _, m := range metrics { - metric := m - metric.LabelKeys = []string{"condition", "status"} - metric.LabelValues = append([]string{string(c.Type)}, metric.LabelValues...) - ms = append(ms, metric) - } - } - - return &metric.Family{ - Metrics: ms, - } - }), - }, - } -) - -func wrapHPAFunc(f func(*autoscaling.HorizontalPodAutoscaler) *metric.Family) func(interface{}) *metric.Family { - return func(obj interface{}) *metric.Family { - hpa := obj.(*autoscaling.HorizontalPodAutoscaler) - - metricFamily := f(hpa) - - for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descHorizontalPodAutoscalerLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{hpa.Namespace, hpa.Name}, m.LabelValues...) - } - - return metricFamily - } -} - -func createHPAListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { - return &cache.ListWatch{ - ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.AutoscalingV2beta1().HorizontalPodAutoscalers(ns).List(opts) - }, - WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.AutoscalingV2beta1().HorizontalPodAutoscalers(ns).Watch(opts) - }, - } -} diff --git a/internal/store/hpa_test.go b/internal/store/hpa_test.go deleted file mode 100644 index 7f4e6bce4e..0000000000 --- a/internal/store/hpa_test.go +++ /dev/null @@ -1,326 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package store - -import ( - "testing" - - autoscaling "k8s.io/api/autoscaling/v2beta1" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "k8s.io/kube-state-metrics/pkg/metric" -) - -var ( - hpa1MinReplicas int32 = 2 -) - -func TestHPAStore(t *testing.T) { - // Fixed metadata on type and help text. We prepend this to every expected - // output so we only have to modify a single place when doing adjustments. - const metadata = ` - # HELP kube_hpa_labels Kubernetes labels converted to Prometheus labels. - # HELP kube_hpa_metadata_generation The generation observed by the HorizontalPodAutoscaler controller. - # HELP kube_hpa_spec_max_replicas Upper limit for the number of pods that can be set by the autoscaler; cannot be smaller than MinReplicas. - # HELP kube_hpa_spec_min_replicas Lower limit for the number of pods that can be set by the autoscaler, default 1. - # HELP kube_hpa_spec_target_metric The metric specifications used by this autoscaler when calculating the desired replica count. - # HELP kube_hpa_status_condition The condition of this autoscaler. - # HELP kube_hpa_status_current_replicas Current number of replicas of pods managed by this autoscaler. - # HELP kube_hpa_status_desired_replicas Desired number of replicas of pods managed by this autoscaler. - # TYPE kube_hpa_labels gauge - # TYPE kube_hpa_metadata_generation gauge - # TYPE kube_hpa_spec_max_replicas gauge - # TYPE kube_hpa_spec_min_replicas gauge - # TYPE kube_hpa_spec_target_metric gauge - # TYPE kube_hpa_status_condition gauge - # TYPE kube_hpa_status_current_replicas gauge - # TYPE kube_hpa_status_desired_replicas gauge - ` - cases := []generateMetricsTestCase{ - { - // Verify populating base metric. - Obj: &autoscaling.HorizontalPodAutoscaler{ - ObjectMeta: metav1.ObjectMeta{ - Generation: 2, - Name: "hpa1", - Namespace: "ns1", - Labels: map[string]string{ - "app": "foobar", - }, - }, - Spec: autoscaling.HorizontalPodAutoscalerSpec{ - MaxReplicas: 4, - MinReplicas: &hpa1MinReplicas, - Metrics: []autoscaling.MetricSpec{ - { - Type: autoscaling.ObjectMetricSourceType, - Object: &autoscaling.ObjectMetricSource{ - MetricName: "hits", - TargetValue: resource.MustParse("10"), - AverageValue: resourcePtr(resource.MustParse("12")), - }, - }, - { - Type: autoscaling.PodsMetricSourceType, - Pods: &autoscaling.PodsMetricSource{ - MetricName: "transactions_processed", - TargetAverageValue: resource.MustParse("33"), - }, - }, - { - Type: autoscaling.ResourceMetricSourceType, - Resource: &autoscaling.ResourceMetricSource{ - Name: "cpu", - TargetAverageUtilization: int32ptr(80), - }, - }, - { - Type: autoscaling.ResourceMetricSourceType, - Resource: &autoscaling.ResourceMetricSource{ - Name: "memory", - TargetAverageUtilization: int32ptr(80), - TargetAverageValue: resourcePtr(resource.MustParse("800Ki")), - }, - }, - // No targets, this metric should be ignored - { - Type: autoscaling.ResourceMetricSourceType, - Resource: &autoscaling.ResourceMetricSource{ - Name: "disk", - }, - }, - { - Type: autoscaling.ExternalMetricSourceType, - External: &autoscaling.ExternalMetricSource{ - MetricName: "sqs_jobs", - TargetValue: resourcePtr(resource.MustParse("30")), - }, - }, - { - Type: autoscaling.ExternalMetricSourceType, - External: &autoscaling.ExternalMetricSource{ - MetricName: "events", - TargetAverageValue: resourcePtr(resource.MustParse("30")), - }, - }, - }, - ScaleTargetRef: autoscaling.CrossVersionObjectReference{ - APIVersion: "apps/v1", - Kind: "Deployment", - Name: "deployment1", - }, - }, - Status: autoscaling.HorizontalPodAutoscalerStatus{ - CurrentReplicas: 2, - DesiredReplicas: 2, - Conditions: []autoscaling.HorizontalPodAutoscalerCondition{ - { - Type: autoscaling.AbleToScale, - Status: v1.ConditionTrue, - Reason: "reason", - }, - }, - CurrentMetrics: []autoscaling.MetricStatus{ - { - Type: "Resource", - Resource: &autoscaling.ResourceMetricStatus{ - Name: "cpu", - CurrentAverageUtilization: new(int32), - CurrentAverageValue: resource.MustParse("7m"), - }, - }, - { - Type: "Resource", - Resource: &autoscaling.ResourceMetricStatus{ - Name: "memory", - CurrentAverageUtilization: new(int32), - CurrentAverageValue: resource.MustParse("26335914666m"), - }, - }, - }, - }, - }, - Want: metadata + ` - kube_hpa_labels{hpa="hpa1",label_app="foobar",namespace="ns1"} 1 - kube_hpa_metadata_generation{hpa="hpa1",namespace="ns1"} 2 - kube_hpa_spec_max_replicas{hpa="hpa1",namespace="ns1"} 4 - kube_hpa_spec_min_replicas{hpa="hpa1",namespace="ns1"} 2 - kube_hpa_spec_target_metric{hpa="hpa1",metric_name="cpu",metric_target_type="utilization",namespace="ns1"} 80 - kube_hpa_spec_target_metric{hpa="hpa1",metric_name="events",metric_target_type="average",namespace="ns1"} 30 - kube_hpa_spec_target_metric{hpa="hpa1",metric_name="hits",metric_target_type="average",namespace="ns1"} 12 - kube_hpa_spec_target_metric{hpa="hpa1",metric_name="hits",metric_target_type="value",namespace="ns1"} 10 - kube_hpa_spec_target_metric{hpa="hpa1",metric_name="memory",metric_target_type="average",namespace="ns1"} 819200 - kube_hpa_spec_target_metric{hpa="hpa1",metric_name="memory",metric_target_type="utilization",namespace="ns1"} 80 - kube_hpa_spec_target_metric{hpa="hpa1",metric_name="sqs_jobs",metric_target_type="value",namespace="ns1"} 30 - kube_hpa_spec_target_metric{hpa="hpa1",metric_name="transactions_processed",metric_target_type="average",namespace="ns1"} 33 - kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="false"} 0 - kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="true"} 1 - kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="unknown"} 0 - kube_hpa_status_current_replicas{hpa="hpa1",namespace="ns1"} 2 - kube_hpa_status_desired_replicas{hpa="hpa1",namespace="ns1"} 2 - `, - MetricNames: []string{ - "kube_hpa_metadata_generation", - "kube_hpa_spec_max_replicas", - "kube_hpa_spec_min_replicas", - "kube_hpa_spec_target_metric", - "kube_hpa_status_current_replicas", - "kube_hpa_status_desired_replicas", - "kube_hpa_status_condition", - "kube_hpa_labels", - }, - }, - { - // Verify populating base metric. - Obj: &autoscaling.HorizontalPodAutoscaler{ - ObjectMeta: metav1.ObjectMeta{ - Generation: 2, - Name: "hpa2", - Namespace: "ns1", - Labels: map[string]string{ - "app": "foobar", - }, - }, - Spec: autoscaling.HorizontalPodAutoscalerSpec{ - MaxReplicas: 4, - MinReplicas: &hpa1MinReplicas, - Metrics: []autoscaling.MetricSpec{ - { - Type: autoscaling.ResourceMetricSourceType, - Resource: &autoscaling.ResourceMetricSource{ - Name: "memory", - TargetAverageUtilization: int32ptr(75), - }, - }, - { - Type: autoscaling.ResourceMetricSourceType, - Resource: &autoscaling.ResourceMetricSource{ - Name: "cpu", - TargetAverageUtilization: int32ptr(80), - }, - }, - { - Type: autoscaling.ExternalMetricSourceType, - External: &autoscaling.ExternalMetricSource{ - MetricName: "traefik_backend_requests_per_second", - TargetValue: resourcePtr(resource.MustParse("100")), - }, - }, - { - Type: autoscaling.ExternalMetricSourceType, - External: &autoscaling.ExternalMetricSource{ - MetricName: "traefik_backend_errors_per_second", - TargetValue: resourcePtr(resource.MustParse("100")), - }, - }, - }, - ScaleTargetRef: autoscaling.CrossVersionObjectReference{ - APIVersion: "apps/v1", - Kind: "Deployment", - Name: "deployment1", - }, - }, - Status: autoscaling.HorizontalPodAutoscalerStatus{ - CurrentReplicas: 2, - DesiredReplicas: 2, - Conditions: []autoscaling.HorizontalPodAutoscalerCondition{ - { - Type: autoscaling.AbleToScale, - Status: v1.ConditionTrue, - Reason: "reason", - }, - }, - CurrentMetrics: []autoscaling.MetricStatus{ - { - Type: "Resource", - Resource: &autoscaling.ResourceMetricStatus{ - Name: "memory", - CurrentAverageUtilization: int32ptr(28), - CurrentAverageValue: resource.MustParse("847775744"), - }, - }, - { - Type: "Resource", - Resource: &autoscaling.ResourceMetricStatus{ - Name: "cpu", - CurrentAverageUtilization: int32ptr(6), - CurrentAverageValue: resource.MustParse("62m"), - }, - }, - { - Type: "External", - External: &autoscaling.ExternalMetricStatus{ - MetricName: "traefik_backend_requests_per_second", - CurrentValue: resource.MustParse("0"), - CurrentAverageValue: resourcePtr(resource.MustParse("2900m")), - }, - }, - { - Type: "External", - External: &autoscaling.ExternalMetricStatus{ - MetricName: "traefik_backend_errors_per_second", - CurrentValue: resource.MustParse("0"), - }, - }, - }, - }, - }, - Want: metadata + ` - kube_hpa_labels{hpa="hpa2",label_app="foobar",namespace="ns1"} 1 - kube_hpa_metadata_generation{hpa="hpa2",namespace="ns1"} 2 - kube_hpa_spec_max_replicas{hpa="hpa2",namespace="ns1"} 4 - kube_hpa_spec_min_replicas{hpa="hpa2",namespace="ns1"} 2 - kube_hpa_spec_target_metric{hpa="hpa2",metric_name="cpu",metric_target_type="utilization",namespace="ns1"} 80 - kube_hpa_spec_target_metric{hpa="hpa2",metric_name="memory",metric_target_type="utilization",namespace="ns1"} 75 - kube_hpa_spec_target_metric{hpa="hpa2",metric_name="traefik_backend_errors_per_second",metric_target_type="value",namespace="ns1"} 100 - kube_hpa_spec_target_metric{hpa="hpa2",metric_name="traefik_backend_requests_per_second",metric_target_type="value",namespace="ns1"} 100 - kube_hpa_status_condition{condition="AbleToScale",hpa="hpa2",namespace="ns1",status="false"} 0 - kube_hpa_status_condition{condition="AbleToScale",hpa="hpa2",namespace="ns1",status="true"} 1 - kube_hpa_status_condition{condition="AbleToScale",hpa="hpa2",namespace="ns1",status="unknown"} 0 - kube_hpa_status_current_replicas{hpa="hpa2",namespace="ns1"} 2 - kube_hpa_status_desired_replicas{hpa="hpa2",namespace="ns1"} 2 - `, - MetricNames: []string{ - "kube_hpa_metadata_generation", - "kube_hpa_spec_max_replicas", - "kube_hpa_spec_min_replicas", - "kube_hpa_spec_target_metric", - "kube_hpa_status_current_replicas", - "kube_hpa_status_desired_replicas", - "kube_hpa_status_condition", - "kube_hpa_labels", - }, - }, - } - for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(hpaMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(hpaMetricFamilies) - if err := c.run(); err != nil { - t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) - } - } -} - -func int32ptr(value int32) *int32 { - return &value -} - -func resourcePtr(quantity resource.Quantity) *resource.Quantity { - return &quantity -} diff --git a/internal/store/ingress.go b/internal/store/ingress.go index f5e7cfaa18..70f9b2bfd5 100644 --- a/internal/store/ingress.go +++ b/internal/store/ingress.go @@ -17,10 +17,15 @@ limitations under the License. package store import ( - "k8s.io/kube-state-metrics/pkg/metric" + "context" + "strconv" - "k8s.io/api/extensions/v1beta1" + basemetrics "k8s.io/component-base/metrics" + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" + + networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" @@ -29,30 +34,67 @@ import ( ) var ( + descIngressAnnotationsName = "kube_ingress_annotations" + descIngressAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." descIngressLabelsName = "kube_ingress_labels" descIngressLabelsHelp = "Kubernetes labels converted to Prometheus labels." descIngressLabelsDefaultLabels = []string{"namespace", "ingress"} +) + +func ingressMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_ingress_info", + "Information about ingress.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapIngressFunc(func(i *networkingv1.Ingress) *metric.Family { + ingressClassName := "_default" + if i.Spec.IngressClassName != nil { + ingressClassName = *i.Spec.IngressClassName + } + if className, ok := i.Annotations["kubernetes.io/ingress.class"]; ok { + ingressClassName = className + } - ingressMetricFamilies = []metric.FamilyGenerator{ - { - Name: "kube_ingress_info", - Type: metric.Gauge, - Help: "Information about ingress.", - GenerateFunc: wrapIngressFunc(func(s *v1beta1.Ingress) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { - Value: 1, + LabelKeys: []string{"ingressclass"}, + LabelValues: []string{ingressClassName}, + Value: 1, }, }} }), - }, - { - Name: descIngressLabelsName, - Type: metric.Gauge, - Help: descIngressLabelsHelp, - GenerateFunc: wrapIngressFunc(func(i *v1beta1.Ingress) *metric.Family { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(i.Labels) + ), + *generator.NewFamilyGeneratorWithStability( + descIngressAnnotationsName, + descIngressAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapIngressFunc(func(i *networkingv1.Ingress) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", i.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }} + + }), + ), + *generator.NewFamilyGeneratorWithStability( + descIngressLabelsName, + descIngressLabelsHelp, + metric.Gauge, + basemetrics.STABLE, + "", + wrapIngressFunc(func(i *networkingv1.Ingress) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", i.Labels, allowLabelsList) return &metric.Family{ Metrics: []*metric.Metric{ { @@ -63,12 +105,14 @@ var ( }} }), - }, - { - Name: "kube_ingress_created", - Type: metric.Gauge, - Help: "Unix creation timestamp", - GenerateFunc: wrapIngressFunc(func(i *v1beta1.Ingress) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_ingress_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.STABLE, + "", + wrapIngressFunc(func(i *networkingv1.Ingress) *metric.Family { ms := []*metric.Metric{} if !i.CreationTimestamp.IsZero() { @@ -81,31 +125,43 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_ingress_metadata_resource_version", - Type: metric.Gauge, - Help: "Resource version representing a specific version of ingress.", - GenerateFunc: wrapIngressFunc(func(i *v1beta1.Ingress) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_ingress_metadata_resource_version", + "Resource version representing a specific version of ingress.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapIngressFunc(func(i *networkingv1.Ingress) *metric.Family { return &metric.Family{ Metrics: resourceVersionMetric(i.ObjectMeta.ResourceVersion), } }), - }, - { - Name: "kube_ingress_path", - Type: metric.Gauge, - Help: "Ingress host, paths and backend service information.", - GenerateFunc: wrapIngressFunc(func(i *v1beta1.Ingress) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_ingress_path", + "Ingress host, paths and backend service information.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapIngressFunc(func(i *networkingv1.Ingress) *metric.Family { ms := []*metric.Metric{} for _, rule := range i.Spec.Rules { if rule.HTTP != nil { for _, path := range rule.HTTP.Paths { - ms = append(ms, &metric.Metric{ - LabelKeys: []string{"host", "path", "service_name", "service_port"}, - LabelValues: []string{rule.Host, path.Path, path.Backend.ServiceName, path.Backend.ServicePort.String()}, - Value: 1, - }) + if path.Backend.Service != nil { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"host", "path", "service_name", "service_port"}, + LabelValues: []string{rule.Host, path.Path, path.Backend.Service.Name, strconv.Itoa(int(path.Backend.Service.Port.Number))}, + Value: 1, + }) + } else { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"host", "path", "service_name", "service_port"}, + LabelValues: []string{rule.Host, path.Path, "", ""}, + Value: 1, + }) + } } } } @@ -113,12 +169,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_ingress_tls", - Type: metric.Gauge, - Help: "Ingress TLS host and secret information.", - GenerateFunc: wrapIngressFunc(func(i *v1beta1.Ingress) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_ingress_tls", + "Ingress TLS host and secret information.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapIngressFunc(func(i *networkingv1.Ingress) *metric.Family { ms := []*metric.Metric{} for _, tls := range i.Spec.TLS { for _, host := range tls.Hosts { @@ -133,32 +191,33 @@ var ( Metrics: ms, } }), - }, + ), } -) +} -func wrapIngressFunc(f func(*v1beta1.Ingress) *metric.Family) func(interface{}) *metric.Family { +func wrapIngressFunc(f func(*networkingv1.Ingress) *metric.Family) func(interface{}) *metric.Family { return func(obj interface{}) *metric.Family { - ingress := obj.(*v1beta1.Ingress) + ingress := obj.(*networkingv1.Ingress) metricFamily := f(ingress) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descIngressLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{ingress.Namespace, ingress.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descIngressLabelsDefaultLabels, []string{ingress.Namespace, ingress.Name}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createIngressListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createIngressListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.ExtensionsV1beta1().Ingresses(ns).List(opts) + opts.FieldSelector = fieldSelector + return kubeClient.NetworkingV1().Ingresses(ns).List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.ExtensionsV1beta1().Ingresses(ns).Watch(opts) + opts.FieldSelector = fieldSelector + return kubeClient.NetworkingV1().Ingresses(ns).Watch(context.TODO(), opts) }, } } diff --git a/internal/store/ingress_test.go b/internal/store/ingress_test.go index e71a99e293..123a679cbd 100644 --- a/internal/store/ingress_test.go +++ b/internal/store/ingress_test.go @@ -19,26 +19,27 @@ package store import ( "testing" - "k8s.io/api/extensions/v1beta1" + v1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) func TestIngressStore(t *testing.T) { startTime := 1501569018 metav1StartTime := metav1.Unix(int64(startTime), 0) + testIngressClass := "test" // Fixed metadata on type and help text. We prepend this to every expected // output so we only have to modify a single place when doing adjustments. const metadata = ` - # HELP kube_ingress_created Unix creation timestamp - # HELP kube_ingress_info Information about ingress. - # HELP kube_ingress_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_ingress_created [STABLE] Unix creation timestamp + # HELP kube_ingress_info [STABLE] Information about ingress. + # HELP kube_ingress_labels [STABLE] Kubernetes labels converted to Prometheus labels. # HELP kube_ingress_metadata_resource_version Resource version representing a specific version of ingress. - # HELP kube_ingress_path Ingress host, paths and backend service information. - # HELP kube_ingress_tls Ingress TLS host and secret information. + # HELP kube_ingress_path [STABLE] Ingress host, paths and backend service information. + # HELP kube_ingress_tls [STABLE] Ingress TLS host and secret information. # TYPE kube_ingress_created gauge # TYPE kube_ingress_info gauge # TYPE kube_ingress_labels gauge @@ -48,22 +49,43 @@ func TestIngressStore(t *testing.T) { ` cases := []generateMetricsTestCase{ { - Obj: &v1beta1.Ingress{ + AllowAnnotationsList: []string{ + "app.k8s.io/owner", + }, + Obj: &networkingv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: "ingress1", Namespace: "ns1", ResourceVersion: "000000", + Annotations: map[string]string{ + "app": "mysql-server", + "app.k8s.io/owner": "@foo", + }, }, }, - Want: metadata + ` - kube_ingress_info{namespace="ns1",ingress="ingress1"} 1 + Want: ` + # HELP kube_ingress_info [STABLE] Information about ingress. + # HELP kube_ingress_annotations Kubernetes annotations converted to Prometheus labels. + # HELP kube_ingress_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # HELP kube_ingress_metadata_resource_version Resource version representing a specific version of ingress. + # TYPE kube_ingress_info gauge + # TYPE kube_ingress_annotations gauge + # TYPE kube_ingress_labels gauge + # TYPE kube_ingress_metadata_resource_version gauge + kube_ingress_info{namespace="ns1",ingress="ingress1",ingressclass="_default"} 1 kube_ingress_metadata_resource_version{namespace="ns1",ingress="ingress1"} 0 + kube_ingress_annotations{annotation_app_k8s_io_owner="@foo",namespace="ns1",ingress="ingress1"} 1 kube_ingress_labels{namespace="ns1",ingress="ingress1"} 1 `, - MetricNames: []string{"kube_ingress_info", "kube_ingress_metadata_resource_version", "kube_ingress_created", "kube_ingress_labels", "kube_ingress_path", "kube_ingress_tls"}, + MetricNames: []string{ + "kube_ingress_info", + "kube_ingress_metadata_resource_version", + "kube_ingress_annotations", + "kube_ingress_labels", + }, }, { - Obj: &v1beta1.Ingress{ + Obj: &networkingv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: "ingress2", Namespace: "ns2", @@ -72,7 +94,7 @@ func TestIngressStore(t *testing.T) { }, }, Want: metadata + ` - kube_ingress_info{namespace="ns2",ingress="ingress2"} 1 + kube_ingress_info{namespace="ns2",ingress="ingress2",ingressclass="_default"} 1 kube_ingress_created{namespace="ns2",ingress="ingress2"} 1.501569018e+09 kube_ingress_metadata_resource_version{namespace="ns2",ingress="ingress2"} 123456 kube_ingress_labels{namespace="ns2",ingress="ingress2"} 1 @@ -80,7 +102,7 @@ func TestIngressStore(t *testing.T) { MetricNames: []string{"kube_ingress_info", "kube_ingress_metadata_resource_version", "kube_ingress_created", "kube_ingress_labels", "kube_ingress_path", "kube_ingress_tls"}, }, { - Obj: &v1beta1.Ingress{ + Obj: &networkingv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: "ingress3", Namespace: "ns3", @@ -90,14 +112,14 @@ func TestIngressStore(t *testing.T) { }, }, Want: metadata + ` - kube_ingress_info{namespace="ns3",ingress="ingress3"} 1 + kube_ingress_info{namespace="ns3",ingress="ingress3",ingressclass="_default"} 1 kube_ingress_created{namespace="ns3",ingress="ingress3"} 1.501569018e+09 - kube_ingress_labels{label_test_3="test-3",namespace="ns3",ingress="ingress3"} 1 + kube_ingress_labels{namespace="ns3",ingress="ingress3"} 1 `, MetricNames: []string{"kube_ingress_info", "kube_ingress_metadata_resource_version", "kube_ingress_created", "kube_ingress_labels", "kube_ingress_path", "kube_ingress_tls"}, }, { - Obj: &v1beta1.Ingress{ + Obj: &networkingv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: "ingress4", Namespace: "ns4", @@ -105,18 +127,31 @@ func TestIngressStore(t *testing.T) { Labels: map[string]string{"test-4": "test-4"}, ResourceVersion: "abcdef", }, - Spec: v1beta1.IngressSpec{ - Rules: []v1beta1.IngressRule{ + Spec: networkingv1.IngressSpec{ + Rules: []networkingv1.IngressRule{ { Host: "somehost", - IngressRuleValue: v1beta1.IngressRuleValue{ - HTTP: &v1beta1.HTTPIngressRuleValue{ - Paths: []v1beta1.HTTPIngressPath{ + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{ { Path: "/somepath", - Backend: v1beta1.IngressBackend{ - ServiceName: "someservice", - ServicePort: intstr.FromInt(1234), + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "someservice", + Port: networkingv1.ServiceBackendPort{ + Number: 1234, + }, + }, + }, + }, + { + Path: "/somepath2", + Backend: networkingv1.IngressBackend{ + Resource: &v1.TypedLocalObjectReference{ + Kind: "somekind", + Name: "somename", + }, }, }, }, @@ -130,15 +165,16 @@ func TestIngressStore(t *testing.T) { }, }, Want: metadata + ` - kube_ingress_info{namespace="ns4",ingress="ingress4"} 1 + kube_ingress_info{namespace="ns4",ingress="ingress4",ingressclass="_default"} 1 kube_ingress_created{namespace="ns4",ingress="ingress4"} 1.501569018e+09 - kube_ingress_labels{label_test_4="test-4",namespace="ns4",ingress="ingress4"} 1 + kube_ingress_labels{namespace="ns4",ingress="ingress4"} 1 kube_ingress_path{namespace="ns4",ingress="ingress4",host="somehost",path="/somepath",service_name="someservice",service_port="1234"} 1 + kube_ingress_path{namespace="ns4",ingress="ingress4",host="somehost",path="/somepath2",service_name="",service_port=""} 1 `, MetricNames: []string{"kube_ingress_info", "kube_ingress_metadata_resource_version", "kube_ingress_created", "kube_ingress_labels", "kube_ingress_path", "kube_ingress_tls"}, }, { - Obj: &v1beta1.Ingress{ + Obj: &networkingv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: "ingress5", Namespace: "ns5", @@ -146,8 +182,8 @@ func TestIngressStore(t *testing.T) { Labels: map[string]string{"test-5": "test-5"}, ResourceVersion: "abcdef", }, - Spec: v1beta1.IngressSpec{ - TLS: []v1beta1.IngressTLS{ + Spec: networkingv1.IngressSpec{ + TLS: []networkingv1.IngressTLS{ { Hosts: []string{"somehost1", "somehost2"}, SecretName: "somesecret", @@ -156,18 +192,58 @@ func TestIngressStore(t *testing.T) { }, }, Want: metadata + ` - kube_ingress_info{namespace="ns5",ingress="ingress5"} 1 + kube_ingress_info{namespace="ns5",ingress="ingress5",ingressclass="_default"} 1 kube_ingress_created{namespace="ns5",ingress="ingress5"} 1.501569018e+09 - kube_ingress_labels{label_test_5="test-5",namespace="ns5",ingress="ingress5"} 1 + kube_ingress_labels{namespace="ns5",ingress="ingress5"} 1 kube_ingress_tls{namespace="ns5",ingress="ingress5",tls_host="somehost1",secret="somesecret"} 1 kube_ingress_tls{namespace="ns5",ingress="ingress5",tls_host="somehost2",secret="somesecret"} 1 `, MetricNames: []string{"kube_ingress_info", "kube_ingress_metadata_resource_version", "kube_ingress_created", "kube_ingress_labels", "kube_ingress_path", "kube_ingress_tls"}, }, + { + Obj: &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ingress6", + Namespace: "ns6", + CreationTimestamp: metav1StartTime, + ResourceVersion: "123456", + }, + Spec: networkingv1.IngressSpec{ + IngressClassName: &testIngressClass, + }, + }, + Want: metadata + ` + kube_ingress_info{namespace="ns6",ingress="ingress6",ingressclass="test"} 1 + kube_ingress_created{namespace="ns6",ingress="ingress6"} 1.501569018e+09 + kube_ingress_metadata_resource_version{namespace="ns6",ingress="ingress6"} 123456 + kube_ingress_labels{namespace="ns6",ingress="ingress6"} 1 + `, + MetricNames: []string{"kube_ingress_info", "kube_ingress_metadata_resource_version", "kube_ingress_created", "kube_ingress_labels", "kube_ingress_path", "kube_ingress_tls"}, + }, + { + Obj: &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ingress7", + Namespace: "ns7", + CreationTimestamp: metav1StartTime, + ResourceVersion: "123456", + Annotations: map[string]string{ + "kubernetes.io/ingress.class": "test", + }, + }, + }, + Want: metadata + ` + kube_ingress_info{namespace="ns7",ingress="ingress7",ingressclass="test"} 1 + kube_ingress_created{namespace="ns7",ingress="ingress7"} 1.501569018e+09 + kube_ingress_metadata_resource_version{namespace="ns7",ingress="ingress7"} 123456 + kube_ingress_labels{namespace="ns7",ingress="ingress7"} 1 + `, + MetricNames: []string{"kube_ingress_info", "kube_ingress_metadata_resource_version", "kube_ingress_created", "kube_ingress_labels", "kube_ingress_path", "kube_ingress_tls"}, + }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(ingressMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(ingressMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(ingressMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) + c.Headers = generator.ExtractMetricFamilyHeaders(ingressMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/ingressclass.go b/internal/store/ingressclass.go new file mode 100644 index 0000000000..3e67881ef0 --- /dev/null +++ b/internal/store/ingressclass.go @@ -0,0 +1,140 @@ +/* +Copyright 2022 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "context" + + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" + + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" +) + +var ( + descIngressClassAnnotationsName = "kube_ingressclass_annotations" + descIngressClassAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." + descIngressClassLabelsName = "kube_ingressclass_labels" + descIngressClassLabelsHelp = "Kubernetes labels converted to Prometheus labels." + descIngressClassLabelsDefaultLabels = []string{"ingressclass"} +) + +func ingressClassMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_ingressclass_info", + "Information about ingressclass.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapIngressClassFunc(func(s *networkingv1.IngressClass) *metric.Family { + + m := metric.Metric{ + LabelKeys: []string{"controller"}, + LabelValues: []string{s.Spec.Controller}, + Value: 1, + } + return &metric.Family{Metrics: []*metric.Metric{&m}} + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_ingressclass_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapIngressClassFunc(func(s *networkingv1.IngressClass) *metric.Family { + ms := []*metric.Metric{} + if !s.CreationTimestamp.IsZero() { + ms = append(ms, &metric.Metric{ + Value: float64(s.CreationTimestamp.Unix()), + }) + } + return &metric.Family{ + Metrics: ms, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descIngressClassAnnotationsName, + descIngressClassAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapIngressClassFunc(func(s *networkingv1.IngressClass) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", s.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descIngressClassLabelsName, + descIngressClassLabelsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapIngressClassFunc(func(s *networkingv1.IngressClass) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", s.Labels, allowLabelsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: labelKeys, + LabelValues: labelValues, + Value: 1, + }, + }, + } + }), + ), + } +} + +func wrapIngressClassFunc(f func(*networkingv1.IngressClass) *metric.Family) func(interface{}) *metric.Family { + return func(obj interface{}) *metric.Family { + ingressClass := obj.(*networkingv1.IngressClass) + + metricFamily := f(ingressClass) + + for _, m := range metricFamily.Metrics { + m.LabelKeys, m.LabelValues = mergeKeyValues(descIngressClassLabelsDefaultLabels, []string{ingressClass.Name}, m.LabelKeys, m.LabelValues) + } + + return metricFamily + } +} + +func createIngressClassListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { + return &cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + return kubeClient.NetworkingV1().IngressClasses().List(context.TODO(), opts) + }, + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + return kubeClient.NetworkingV1().IngressClasses().Watch(context.TODO(), opts) + }, + } +} diff --git a/internal/store/ingressclass_test.go b/internal/store/ingressclass_test.go new file mode 100644 index 0000000000..50312d0ca3 --- /dev/null +++ b/internal/store/ingressclass_test.go @@ -0,0 +1,105 @@ +/* +Copyright 2022 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "testing" + + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +func TestIngressClassStore(t *testing.T) { + startTime := 1501569018 + metav1StartTime := metav1.Unix(int64(startTime), 0) + + cases := []generateMetricsTestCase{ + { + Obj: &networkingv1.IngressClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test_ingressclass-info", + }, + Spec: networkingv1.IngressClassSpec{ + Controller: "controller", + }, + }, + Want: ` + # HELP kube_ingressclass_info Information about ingressclass. + # TYPE kube_ingressclass_info gauge + kube_ingressclass_info{ingressclass="test_ingressclass-info",controller="controller"} 1 + `, + MetricNames: []string{ + "kube_ingressclass_info", + }, + }, + { + Obj: &networkingv1.IngressClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test_kube_ingressclass-created", + CreationTimestamp: metav1StartTime, + }, + Spec: networkingv1.IngressClassSpec{ + Controller: "controller", + }, + }, + Want: ` + # HELP kube_ingressclass_created Unix creation timestamp + # TYPE kube_ingressclass_created gauge + kube_ingressclass_created{ingressclass="test_kube_ingressclass-created"} 1.501569018e+09 + `, + MetricNames: []string{ + "kube_ingressclass_created", + }, + }, + { + AllowAnnotationsList: []string{ + "ingressclass.kubernetes.io/is-default-class", + }, + Obj: &networkingv1.IngressClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test_ingressclass-labels", + Annotations: map[string]string{ + "ingressclass.kubernetes.io/is-default-class": "true", + }, + Labels: map[string]string{ + "foo": "bar", + }, + }, + Spec: networkingv1.IngressClassSpec{ + Controller: "controller", + }, + }, + Want: ` + # HELP kube_ingressclass_annotations Kubernetes annotations converted to Prometheus labels. + # HELP kube_ingressclass_labels Kubernetes labels converted to Prometheus labels. + # TYPE kube_ingressclass_annotations gauge + # TYPE kube_ingressclass_labels gauge + kube_ingressclass_annotations{ingressclass="test_ingressclass-labels",annotation_ingressclass_kubernetes_io_is_default_class="true"} 1 + kube_ingressclass_labels{ingressclass="test_ingressclass-labels"} 1 + `, + MetricNames: []string{ + "kube_ingressclass_annotations", "kube_ingressclass_labels", + }, + }, + } + for i, c := range cases { + c.Func = generator.ComposeMetricGenFuncs(ingressClassMetricFamilies(c.AllowAnnotationsList, nil)) + c.Headers = generator.ExtractMetricFamilyHeaders(ingressClassMetricFamilies(c.AllowAnnotationsList, nil)) + if err := c.run(); err != nil { + t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) + } + } +} diff --git a/internal/store/job.go b/internal/store/job.go index 57c79113a6..19f59cc179 100644 --- a/internal/store/job.go +++ b/internal/store/job.go @@ -17,9 +17,13 @@ limitations under the License. package store import ( + "context" "strconv" - "k8s.io/kube-state-metrics/pkg/metric" + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" v1batch "k8s.io/api/batch/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -30,17 +34,43 @@ import ( ) var ( + descJobAnnotationsName = "kube_job_annotations" + descJobAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." descJobLabelsName = "kube_job_labels" descJobLabelsHelp = "Kubernetes labels converted to Prometheus labels." descJobLabelsDefaultLabels = []string{"namespace", "job_name"} + jobFailureReasons = []string{"BackoffLimitExceeded", "DeadLineExceeded", "Evicted"} +) - jobMetricFamilies = []metric.FamilyGenerator{ - { - Name: descJobLabelsName, - Type: metric.Gauge, - Help: descJobLabelsHelp, - GenerateFunc: wrapJobFunc(func(j *v1batch.Job) *metric.Family { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(j.Labels) +func jobMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + descJobAnnotationsName, + descJobAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapJobFunc(func(j *v1batch.Job) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", j.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descJobLabelsName, + descJobLabelsHelp, + metric.Gauge, + basemetrics.STABLE, + "", + wrapJobFunc(func(j *v1batch.Job) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", j.Labels, allowLabelsList) return &metric.Family{ Metrics: []*metric.Metric{ { @@ -51,12 +81,14 @@ var ( }, } }), - }, - { - Name: "kube_job_info", - Type: metric.Gauge, - Help: "Information about job.", - GenerateFunc: wrapJobFunc(func(j *v1batch.Job) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_job_info", + "Information about job.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapJobFunc(func(j *v1batch.Job) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -65,12 +97,14 @@ var ( }, } }), - }, - { - Name: "kube_job_created", - Type: metric.Gauge, - Help: "Unix creation timestamp", - GenerateFunc: wrapJobFunc(func(j *v1batch.Job) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_job_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.STABLE, + "", + wrapJobFunc(func(j *v1batch.Job) *metric.Family { ms := []*metric.Metric{} if !j.CreationTimestamp.IsZero() { @@ -83,12 +117,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_job_spec_parallelism", - Type: metric.Gauge, - Help: "The maximum desired number of pods the job should run at any given time.", - GenerateFunc: wrapJobFunc(func(j *v1batch.Job) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_job_spec_parallelism", + "The maximum desired number of pods the job should run at any given time.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapJobFunc(func(j *v1batch.Job) *metric.Family { ms := []*metric.Metric{} if j.Spec.Parallelism != nil { @@ -101,12 +137,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_job_spec_completions", - Type: metric.Gauge, - Help: "The desired number of successfully finished pods the job should be run with.", - GenerateFunc: wrapJobFunc(func(j *v1batch.Job) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_job_spec_completions", + "The desired number of successfully finished pods the job should be run with.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapJobFunc(func(j *v1batch.Job) *metric.Family { ms := []*metric.Metric{} if j.Spec.Completions != nil { @@ -119,12 +157,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_job_spec_active_deadline_seconds", - Type: metric.Gauge, - Help: "The duration in seconds relative to the startTime that the job may be active before the system tries to terminate it.", - GenerateFunc: wrapJobFunc(func(j *v1batch.Job) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_job_spec_active_deadline_seconds", + "The duration in seconds relative to the startTime that the job may be active before the system tries to terminate it.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapJobFunc(func(j *v1batch.Job) *metric.Family { ms := []*metric.Metric{} if j.Spec.ActiveDeadlineSeconds != nil { @@ -137,12 +177,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_job_status_succeeded", - Type: metric.Gauge, - Help: "The number of pods which reached Phase Succeeded.", - GenerateFunc: wrapJobFunc(func(j *v1batch.Job) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_job_status_succeeded", + "The number of pods which reached Phase Succeeded.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapJobFunc(func(j *v1batch.Job) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -151,26 +193,63 @@ var ( }, } }), - }, - { - Name: "kube_job_status_failed", - Type: metric.Gauge, - Help: "The number of pods which reached Phase Failed.", - GenerateFunc: wrapJobFunc(func(j *v1batch.Job) *metric.Family { - return &metric.Family{ - Metrics: []*metric.Metric{ - { - Value: float64(j.Status.Failed), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_job_status_failed", + "The number of pods which reached Phase Failed and the reason for failure.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapJobFunc(func(j *v1batch.Job) *metric.Family { + var ms []*metric.Metric + + if float64(j.Status.Failed) == 0 { + return &metric.Family{ + Metrics: []*metric.Metric{ + { + Value: float64(j.Status.Failed), + }, }, - }, + } + } + + for _, c := range j.Status.Conditions { + condition := c + if condition.Type == v1batch.JobFailed { + reasonKnown := false + for _, reason := range jobFailureReasons { + reasonKnown = reasonKnown || failureReason(&condition, reason) + + // for known reasons + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"reason"}, + LabelValues: []string{reason}, + Value: boolFloat64(failureReason(&condition, reason)), + }) + } + // for unknown reasons + if !reasonKnown { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"reason"}, + LabelValues: []string{""}, + Value: float64(j.Status.Failed), + }) + } + } + } + + return &metric.Family{ + Metrics: ms, } }), - }, - { - Name: "kube_job_status_active", - Type: metric.Gauge, - Help: "The number of actively running pods.", - GenerateFunc: wrapJobFunc(func(j *v1batch.Job) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_job_status_active", + "The number of actively running pods.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapJobFunc(func(j *v1batch.Job) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -179,12 +258,14 @@ var ( }, } }), - }, - { - Name: "kube_job_complete", - Type: metric.Gauge, - Help: "The job has completed its execution.", - GenerateFunc: wrapJobFunc(func(j *v1batch.Job) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_job_complete", + "The job has completed its execution.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapJobFunc(func(j *v1batch.Job) *metric.Family { ms := []*metric.Metric{} for _, c := range j.Status.Conditions { if c.Type == v1batch.JobComplete { @@ -201,12 +282,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_job_failed", - Type: metric.Gauge, - Help: "The job has failed its execution.", - GenerateFunc: wrapJobFunc(func(j *v1batch.Job) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_job_failed", + "The job has failed its execution.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapJobFunc(func(j *v1batch.Job) *metric.Family { ms := []*metric.Metric{} for _, c := range j.Status.Conditions { @@ -224,12 +307,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_job_status_start_time", - Type: metric.Gauge, - Help: "StartTime represents time when the job was acknowledged by the Job Manager.", - GenerateFunc: wrapJobFunc(func(j *v1batch.Job) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_job_status_start_time", + "StartTime represents time when the job was acknowledged by the Job Manager.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapJobFunc(func(j *v1batch.Job) *metric.Family { ms := []*metric.Metric{} if j.Status.StartTime != nil { @@ -243,12 +328,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_job_status_completion_time", - Type: metric.Gauge, - Help: "CompletionTime represents time when the job was completed.", - GenerateFunc: wrapJobFunc(func(j *v1batch.Job) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_job_status_completion_time", + "CompletionTime represents time when the job was completed.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapJobFunc(func(j *v1batch.Job) *metric.Family { ms := []*metric.Metric{} if j.Status.CompletionTime != nil { ms = append(ms, &metric.Metric{ @@ -261,12 +348,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_job_owner", - Type: metric.Gauge, - Help: "Information about the Job's owner.", - GenerateFunc: wrapJobFunc(func(j *v1batch.Job) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_job_owner", + "Information about the Job's owner.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapJobFunc(func(j *v1batch.Job) *metric.Family { labelKeys := []string{"owner_kind", "owner_name", "owner_is_controller"} owners := j.GetOwnerReferences() @@ -276,7 +365,7 @@ var ( Metrics: []*metric.Metric{ { LabelKeys: labelKeys, - LabelValues: []string{"", "", ""}, + LabelValues: []string{"", "", ""}, Value: 1, }, }, @@ -305,9 +394,9 @@ var ( Metrics: ms, } }), - }, + ), } -) +} func wrapJobFunc(f func(*v1batch.Job) *metric.Family) func(interface{}) *metric.Family { return func(obj interface{}) *metric.Family { @@ -316,21 +405,29 @@ func wrapJobFunc(f func(*v1batch.Job) *metric.Family) func(interface{}) *metric. metricFamily := f(job) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descJobLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{job.Namespace, job.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descJobLabelsDefaultLabels, []string{job.Namespace, job.Name}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createJobListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createJobListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.BatchV1().Jobs(ns).List(opts) + opts.FieldSelector = fieldSelector + return kubeClient.BatchV1().Jobs(ns).List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.BatchV1().Jobs(ns).Watch(opts) + opts.FieldSelector = fieldSelector + return kubeClient.BatchV1().Jobs(ns).Watch(context.TODO(), opts) }, } } + +func failureReason(jc *v1batch.JobCondition, reason string) bool { + if jc == nil { + return false + } + return jc.Reason == reason +} diff --git a/internal/store/job_test.go b/internal/store/job_test.go index de0cc21472..2394c09091 100644 --- a/internal/store/job_test.go +++ b/internal/store/job_test.go @@ -24,7 +24,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) var ( @@ -48,35 +48,37 @@ func TestJobStore(t *testing.T) { // Fixed metadata on type and help text. We prepend this to every expected // output so we only have to modify a single place when doing adjustments. const metadata = ` - # HELP kube_job_created Unix creation timestamp + # HELP kube_job_annotations Kubernetes annotations converted to Prometheus labels. + # TYPE kube_job_annotations gauge + # HELP kube_job_created [STABLE] Unix creation timestamp # TYPE kube_job_created gauge - # HELP kube_job_owner Information about the Job's owner. + # HELP kube_job_owner [STABLE] Information about the Job's owner. # TYPE kube_job_owner gauge - # HELP kube_job_complete The job has completed its execution. + # HELP kube_job_complete [STABLE] The job has completed its execution. # TYPE kube_job_complete gauge - # HELP kube_job_failed The job has failed its execution. + # HELP kube_job_failed [STABLE] The job has failed its execution. # TYPE kube_job_failed gauge - # HELP kube_job_info Information about job. + # HELP kube_job_info [STABLE] Information about job. # TYPE kube_job_info gauge - # HELP kube_job_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_job_labels [STABLE] Kubernetes labels converted to Prometheus labels. # TYPE kube_job_labels gauge - # HELP kube_job_spec_active_deadline_seconds The duration in seconds relative to the startTime that the job may be active before the system tries to terminate it. + # HELP kube_job_spec_active_deadline_seconds [STABLE] The duration in seconds relative to the startTime that the job may be active before the system tries to terminate it. # TYPE kube_job_spec_active_deadline_seconds gauge - # HELP kube_job_spec_completions The desired number of successfully finished pods the job should be run with. + # HELP kube_job_spec_completions [STABLE] The desired number of successfully finished pods the job should be run with. # TYPE kube_job_spec_completions gauge - # HELP kube_job_spec_parallelism The maximum desired number of pods the job should run at any given time. + # HELP kube_job_spec_parallelism [STABLE] The maximum desired number of pods the job should run at any given time. # TYPE kube_job_spec_parallelism gauge - # HELP kube_job_status_active The number of actively running pods. + # HELP kube_job_status_active [STABLE] The number of actively running pods. # TYPE kube_job_status_active gauge - # HELP kube_job_status_completion_time CompletionTime represents time when the job was completed. + # HELP kube_job_status_completion_time [STABLE] CompletionTime represents time when the job was completed. # TYPE kube_job_status_completion_time gauge - # HELP kube_job_status_failed The number of pods which reached Phase Failed. + # HELP kube_job_status_failed [STABLE] The number of pods which reached Phase Failed and the reason for failure. # TYPE kube_job_status_failed gauge - # HELP kube_job_status_start_time StartTime represents time when the job was acknowledged by the Job Manager. + # HELP kube_job_status_start_time [STABLE] StartTime represents time when the job was acknowledged by the Job Manager. # TYPE kube_job_status_start_time gauge - # HELP kube_job_status_succeeded The number of pods which reached Phase Succeeded. - # TYPE kube_job_status_succeeded gauge - ` + # HELP kube_job_status_succeeded [STABLE] The number of pods which reached Phase Succeeded. + # TYPE kube_job_status_succeeded gauge` + cases := []generateMetricsTestCase{ { Obj: &v1batch.Job{ @@ -110,10 +112,11 @@ func TestJobStore(t *testing.T) { }, }, Want: metadata + ` + kube_job_annotations{job_name="RunningJob1",namespace="ns1"} 1 kube_job_owner{job_name="RunningJob1",namespace="ns1",owner_is_controller="true",owner_kind="CronJob",owner_name="cronjob-name"} 1 kube_job_created{job_name="RunningJob1",namespace="ns1"} 1.5e+09 kube_job_info{job_name="RunningJob1",namespace="ns1"} 1 - kube_job_labels{job_name="RunningJob1",label_app="example-running-1",namespace="ns1"} 1 + kube_job_labels{job_name="RunningJob1",namespace="ns1"} 1 kube_job_spec_active_deadline_seconds{job_name="RunningJob1",namespace="ns1"} 900 kube_job_spec_completions{job_name="RunningJob1",namespace="ns1"} 1 kube_job_spec_parallelism{job_name="RunningJob1",namespace="ns1"} 1 @@ -150,12 +153,13 @@ func TestJobStore(t *testing.T) { }, }, Want: metadata + ` - kube_job_owner{job_name="SuccessfulJob1",namespace="ns1",owner_is_controller="",owner_kind="",owner_name=""} 1 + kube_job_annotations{job_name="SuccessfulJob1",namespace="ns1"} 1 + kube_job_owner{job_name="SuccessfulJob1",namespace="ns1",owner_is_controller="",owner_kind="",owner_name=""} 1 kube_job_complete{condition="false",job_name="SuccessfulJob1",namespace="ns1"} 0 kube_job_complete{condition="true",job_name="SuccessfulJob1",namespace="ns1"} 1 kube_job_complete{condition="unknown",job_name="SuccessfulJob1",namespace="ns1"} 0 kube_job_info{job_name="SuccessfulJob1",namespace="ns1"} 1 - kube_job_labels{job_name="SuccessfulJob1",label_app="example-successful-1",namespace="ns1"} 1 + kube_job_labels{job_name="SuccessfulJob1",namespace="ns1"} 1 kube_job_spec_active_deadline_seconds{job_name="SuccessfulJob1",namespace="ns1"} 900 kube_job_spec_completions{job_name="SuccessfulJob1",namespace="ns1"} 1 kube_job_spec_parallelism{job_name="SuccessfulJob1",namespace="ns1"} 1 @@ -183,7 +187,7 @@ func TestJobStore(t *testing.T) { CompletionTime: &metav1.Time{Time: FailedJob1CompletionTime}, StartTime: &metav1.Time{Time: FailedJob1StartTime}, Conditions: []v1batch.JobCondition{ - {Type: v1batch.JobFailed, Status: v1.ConditionTrue}, + {Type: v1batch.JobFailed, Status: v1.ConditionTrue, Reason: "BackoffLimitExceeded"}, }, }, Spec: v1batch.JobSpec{ @@ -193,18 +197,21 @@ func TestJobStore(t *testing.T) { }, }, Want: metadata + ` - kube_job_owner{job_name="FailedJob1",namespace="ns1",owner_is_controller="",owner_kind="",owner_name=""} 1 + kube_job_annotations{job_name="FailedJob1",namespace="ns1"} 1 + kube_job_owner{job_name="FailedJob1",namespace="ns1",owner_is_controller="",owner_kind="",owner_name=""} 1 kube_job_failed{condition="false",job_name="FailedJob1",namespace="ns1"} 0 kube_job_failed{condition="true",job_name="FailedJob1",namespace="ns1"} 1 kube_job_failed{condition="unknown",job_name="FailedJob1",namespace="ns1"} 0 kube_job_info{job_name="FailedJob1",namespace="ns1"} 1 - kube_job_labels{job_name="FailedJob1",label_app="example-failed-1",namespace="ns1"} 1 + kube_job_labels{job_name="FailedJob1",namespace="ns1"} 1 kube_job_spec_active_deadline_seconds{job_name="FailedJob1",namespace="ns1"} 900 kube_job_spec_completions{job_name="FailedJob1",namespace="ns1"} 1 kube_job_spec_parallelism{job_name="FailedJob1",namespace="ns1"} 1 kube_job_status_active{job_name="FailedJob1",namespace="ns1"} 0 kube_job_status_completion_time{job_name="FailedJob1",namespace="ns1"} 1.495810807e+09 - kube_job_status_failed{job_name="FailedJob1",namespace="ns1"} 1 + kube_job_status_failed{job_name="FailedJob1",namespace="ns1",reason="BackoffLimitExceeded"} 1 + kube_job_status_failed{job_name="FailedJob1",namespace="ns1",reason="DeadLineExceeded"} 0 + kube_job_status_failed{job_name="FailedJob1",namespace="ns1",reason="Evicted"} 0 kube_job_status_start_time{job_name="FailedJob1",namespace="ns1"} 1.495807207e+09 kube_job_status_succeeded{job_name="FailedJob1",namespace="ns1"} 0 `, @@ -236,13 +243,14 @@ func TestJobStore(t *testing.T) { }, }, Want: metadata + ` - kube_job_owner{job_name="SuccessfulJob2NoActiveDeadlineSeconds",namespace="ns1",owner_is_controller="",owner_kind="",owner_name=""} 1 + kube_job_owner{job_name="SuccessfulJob2NoActiveDeadlineSeconds",namespace="ns1",owner_is_controller="",owner_kind="",owner_name=""} 1 kube_job_complete{condition="false",job_name="SuccessfulJob2NoActiveDeadlineSeconds",namespace="ns1"} 0 kube_job_complete{condition="true",job_name="SuccessfulJob2NoActiveDeadlineSeconds",namespace="ns1"} 1 + kube_job_annotations{job_name="SuccessfulJob2NoActiveDeadlineSeconds",namespace="ns1"} 1 kube_job_complete{condition="unknown",job_name="SuccessfulJob2NoActiveDeadlineSeconds",namespace="ns1"} 0 kube_job_info{job_name="SuccessfulJob2NoActiveDeadlineSeconds",namespace="ns1"} 1 - kube_job_labels{job_name="SuccessfulJob2NoActiveDeadlineSeconds",label_app="example-successful-2",namespace="ns1"} 1 + kube_job_labels{job_name="SuccessfulJob2NoActiveDeadlineSeconds",namespace="ns1"} 1 kube_job_spec_completions{job_name="SuccessfulJob2NoActiveDeadlineSeconds",namespace="ns1"} 1 kube_job_spec_parallelism{job_name="SuccessfulJob2NoActiveDeadlineSeconds",namespace="ns1"} 1 kube_job_status_active{job_name="SuccessfulJob2NoActiveDeadlineSeconds",namespace="ns1"} 0 @@ -254,8 +262,8 @@ func TestJobStore(t *testing.T) { }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(jobMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(jobMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(jobMetricFamilies(nil, nil)) + c.Headers = generator.ExtractMetricFamilyHeaders(jobMetricFamilies(nil, nil)) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/lease.go b/internal/store/lease.go new file mode 100644 index 0000000000..5f01f29d0c --- /dev/null +++ b/internal/store/lease.go @@ -0,0 +1,124 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "context" + + coordinationv1 "k8s.io/api/coordination/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +var ( + descLeaseLabelsDefaultLabels = []string{"lease"} + + leaseMetricFamilies = []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_lease_owner", + "Information about the Lease's owner.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapLeaseFunc(func(l *coordinationv1.Lease) *metric.Family { + labelKeys := []string{"owner_kind", "owner_name", "namespace", "lease_holder"} + + var holder string + if l.Spec.HolderIdentity != nil { + holder = *l.Spec.HolderIdentity + } + + owners := l.GetOwnerReferences() + if len(owners) == 0 { + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: labelKeys, + LabelValues: []string{"", "", l.Namespace, holder}, + Value: 1, + }, + }, + } + } + ms := make([]*metric.Metric, len(owners)) + + for i, owner := range owners { + ms[i] = &metric.Metric{ + LabelKeys: labelKeys, + LabelValues: []string{owner.Kind, owner.Name, l.Namespace, holder}, + Value: 1, + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_lease_renew_time", + "Kube lease renew time.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapLeaseFunc(func(l *coordinationv1.Lease) *metric.Family { + ms := []*metric.Metric{} + + if !l.Spec.RenewTime.IsZero() { + ms = append(ms, &metric.Metric{ + Value: float64(l.Spec.RenewTime.Unix()), + }) + } + return &metric.Family{ + Metrics: ms, + } + }), + ), + } +) + +func wrapLeaseFunc(f func(*coordinationv1.Lease) *metric.Family) func(interface{}) *metric.Family { + return func(obj interface{}) *metric.Family { + lease := obj.(*coordinationv1.Lease) + + metricFamily := f(lease) + + for _, m := range metricFamily.Metrics { + m.LabelKeys, m.LabelValues = mergeKeyValues(descLeaseLabelsDefaultLabels, []string{lease.Name}, m.LabelKeys, m.LabelValues) + } + + return metricFamily + } +} + +func createLeaseListWatch(kubeClient clientset.Interface, _ string, _ string) cache.ListerWatcher { + return &cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + return kubeClient.CoordinationV1().Leases("").List(context.TODO(), opts) + }, + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + return kubeClient.CoordinationV1().Leases("").Watch(context.TODO(), opts) + }, + } +} diff --git a/internal/store/lease_test.go b/internal/store/lease_test.go new file mode 100644 index 0000000000..23f26f2403 --- /dev/null +++ b/internal/store/lease_test.go @@ -0,0 +1,103 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "testing" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +func TestLeaseStore(t *testing.T) { + const metadata = ` + # HELP kube_lease_owner Information about the Lease's owner. + # TYPE kube_lease_owner gauge + # HELP kube_lease_renew_time Kube lease renew time. + # TYPE kube_lease_renew_time gauge + ` + leaseOwner := "kube-master" + var ( + cases = []generateMetricsTestCase{ + { + Obj: &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 2, + Name: "kube-master", + Namespace: "default", + CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "Node", + Name: leaseOwner, + }, + }, + }, + Spec: coordinationv1.LeaseSpec{ + RenewTime: &metav1.MicroTime{Time: time.Unix(1500000000, 0)}, + HolderIdentity: &leaseOwner, + }, + }, + Want: metadata + ` + kube_lease_owner{lease="kube-master",owner_kind="Node",owner_name="kube-master",namespace="default",lease_holder="kube-master"} 1 + kube_lease_renew_time{lease="kube-master"} 1.5e+09 + `, + MetricNames: []string{ + "kube_lease_owner", + "kube_lease_renew_time", + }, + }, + { + Obj: &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 2, + Name: "kube-master", + Namespace: "default", + CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "Node", + Name: leaseOwner, + }, + }, + }, + Spec: coordinationv1.LeaseSpec{ + RenewTime: &metav1.MicroTime{Time: time.Unix(1500000000, 0)}, + }, + }, + Want: metadata + ` + kube_lease_owner{lease="kube-master",owner_kind="Node",owner_name="kube-master",namespace="default",lease_holder=""} 1 + kube_lease_renew_time{lease="kube-master"} 1.5e+09 + `, + MetricNames: []string{ + "kube_lease_owner", + "kube_lease_renew_time", + }, + }, + } + ) + for i, c := range cases { + c.Func = generator.ComposeMetricGenFuncs(leaseMetricFamilies) + c.Headers = generator.ExtractMetricFamilyHeaders(leaseMetricFamilies) + if err := c.run(); err != nil { + t.Errorf("unexpected collecting result in %dth run:\n%v", i, err) + } + } +} diff --git a/internal/store/limitrange.go b/internal/store/limitrange.go index 51fc0cc882..ef1bd8ffa4 100644 --- a/internal/store/limitrange.go +++ b/internal/store/limitrange.go @@ -17,25 +17,31 @@ limitations under the License. package store import ( + "context" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" - "k8s.io/kube-state-metrics/pkg/metric" + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) var ( descLimitRangeLabelsDefaultLabels = []string{"namespace", "limitrange"} - limitRangeMetricFamilies = []metric.FamilyGenerator{ - { - Name: "kube_limitrange", - Type: metric.Gauge, - Help: "Information about limit range.", - GenerateFunc: wrapLimitRangeFunc(func(r *v1.LimitRange) *metric.Family { + limitRangeMetricFamilies = []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_limitrange", + "Information about limit range.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapLimitRangeFunc(func(r *v1.LimitRange) *metric.Family { ms := []*metric.Metric{} rawLimitRanges := r.Spec.Limits @@ -84,12 +90,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_limitrange_created", - Type: metric.Gauge, - Help: "Unix creation timestamp", - GenerateFunc: wrapLimitRangeFunc(func(r *v1.LimitRange) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_limitrange_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.STABLE, + "", + wrapLimitRangeFunc(func(r *v1.LimitRange) *metric.Family { ms := []*metric.Metric{} if !r.CreationTimestamp.IsZero() { @@ -103,7 +111,7 @@ var ( Metrics: ms, } }), - }, + ), } ) @@ -114,21 +122,22 @@ func wrapLimitRangeFunc(f func(*v1.LimitRange) *metric.Family) func(interface{}) metricFamily := f(limitRange) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descLimitRangeLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{limitRange.Namespace, limitRange.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descLimitRangeLabelsDefaultLabels, []string{limitRange.Namespace, limitRange.Name}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createLimitRangeListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createLimitRangeListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.CoreV1().LimitRanges(ns).List(opts) + opts.FieldSelector = fieldSelector + return kubeClient.CoreV1().LimitRanges(ns).List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.CoreV1().LimitRanges(ns).Watch(opts) + opts.FieldSelector = fieldSelector + return kubeClient.CoreV1().LimitRanges(ns).Watch(context.TODO(), opts) }, } } diff --git a/internal/store/limitrange_test.go b/internal/store/limitrange_test.go index 3418c7f988..76741e3e16 100644 --- a/internal/store/limitrange_test.go +++ b/internal/store/limitrange_test.go @@ -24,7 +24,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) func TestLimitRangeStore(t *testing.T) { @@ -33,9 +33,9 @@ func TestLimitRangeStore(t *testing.T) { // Fixed metadata on type and help text. We prepend this to every expected // output so we only have to modify a single place when doing adjustments. const metadata = ` - # HELP kube_limitrange_created Unix creation timestamp + # HELP kube_limitrange_created [STABLE] Unix creation timestamp # TYPE kube_limitrange_created gauge - # HELP kube_limitrange Information about limit range. + # HELP kube_limitrange [STABLE] Information about limit range. # TYPE kube_limitrange gauge ` cases := []generateMetricsTestCase{ @@ -81,8 +81,8 @@ func TestLimitRangeStore(t *testing.T) { }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(limitRangeMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(limitRangeMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(limitRangeMetricFamilies) + c.Headers = generator.ExtractMetricFamilyHeaders(limitRangeMetricFamilies) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/mutatingwebhookconfiguration.go b/internal/store/mutatingwebhookconfiguration.go index 57a0d61504..917ca1e799 100644 --- a/internal/store/mutatingwebhookconfiguration.go +++ b/internal/store/mutatingwebhookconfiguration.go @@ -17,26 +17,31 @@ limitations under the License. package store import ( + "context" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" - "k8s.io/kube-state-metrics/pkg/metric" + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) var ( - descMutatingWebhookConfigurationHelp = "Kubernetes labels converted to Prometheus labels." descMutatingWebhookConfigurationDefaultLabels = []string{"namespace", "mutatingwebhookconfiguration"} - mutatingWebhookConfigurationMetricFamilies = []metric.FamilyGenerator{ - { - Name: "kube_mutatingwebhookconfiguration_info", - Type: metric.Gauge, - Help: "Information about the MutatingWebhookConfiguration.", - GenerateFunc: wrapMutatingWebhookConfigurationFunc(func(mwc *admissionregistrationv1.MutatingWebhookConfiguration) *metric.Family { + mutatingWebhookConfigurationMetricFamilies = []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_mutatingwebhookconfiguration_info", + "Information about the MutatingWebhookConfiguration.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapMutatingWebhookConfigurationFunc(func(mwc *admissionregistrationv1.MutatingWebhookConfiguration) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -45,12 +50,14 @@ var ( }, } }), - }, - { - Name: "kube_mutatingwebhookconfiguration_created", - Type: metric.Gauge, - Help: "Unix creation timestamp.", - GenerateFunc: wrapMutatingWebhookConfigurationFunc(func(mwc *admissionregistrationv1.MutatingWebhookConfiguration) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_mutatingwebhookconfiguration_created", + "Unix creation timestamp.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapMutatingWebhookConfigurationFunc(func(mwc *admissionregistrationv1.MutatingWebhookConfiguration) *metric.Family { ms := []*metric.Metric{} if !mwc.CreationTimestamp.IsZero() { @@ -62,27 +69,29 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_mutatingwebhookconfiguration_metadata_resource_version", - Type: metric.Gauge, - Help: "Resource version representing a specific version of the MutatingWebhookConfiguration.", - GenerateFunc: wrapMutatingWebhookConfigurationFunc(func(mwc *admissionregistrationv1.MutatingWebhookConfiguration) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_mutatingwebhookconfiguration_metadata_resource_version", + "Resource version representing a specific version of the MutatingWebhookConfiguration.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapMutatingWebhookConfigurationFunc(func(mwc *admissionregistrationv1.MutatingWebhookConfiguration) *metric.Family { return &metric.Family{ Metrics: resourceVersionMetric(mwc.ObjectMeta.ResourceVersion), } }), - }, + ), } ) -func createMutatingWebhookConfigurationListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createMutatingWebhookConfigurationListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().List(opts) + return kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Watch(opts) + return kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Watch(context.TODO(), opts) }, } } @@ -94,8 +103,7 @@ func wrapMutatingWebhookConfigurationFunc(f func(*admissionregistrationv1.Mutati metricFamily := f(mutatingWebhookConfiguration) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descMutatingWebhookConfigurationDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{mutatingWebhookConfiguration.Namespace, mutatingWebhookConfiguration.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descMutatingWebhookConfigurationDefaultLabels, []string{mutatingWebhookConfiguration.Namespace, mutatingWebhookConfiguration.Name}, m.LabelKeys, m.LabelValues) } return metricFamily diff --git a/internal/store/mutatingwebhookconfiguration_test.go b/internal/store/mutatingwebhookconfiguration_test.go index c78a0ab0fd..325a18ae28 100644 --- a/internal/store/mutatingwebhookconfiguration_test.go +++ b/internal/store/mutatingwebhookconfiguration_test.go @@ -22,7 +22,7 @@ import ( admissionregistrationv1 "k8s.io/api/admissionregistration/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) func TestMutatingWebhookConfigurationStore(t *testing.T) { @@ -71,8 +71,8 @@ func TestMutatingWebhookConfigurationStore(t *testing.T) { }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(mutatingWebhookConfigurationMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(mutatingWebhookConfigurationMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(mutatingWebhookConfigurationMetricFamilies) + c.Headers = generator.ExtractMetricFamilyHeaders(mutatingWebhookConfigurationMetricFamilies) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/namespace.go b/internal/store/namespace.go index 01d04a3b62..bf44f472ae 100644 --- a/internal/store/namespace.go +++ b/internal/store/namespace.go @@ -17,7 +17,12 @@ limitations under the License. package store import ( - "k8s.io/kube-state-metrics/pkg/metric" + "context" + + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -28,16 +33,22 @@ import ( ) var ( + descNamespaceAnnotationsName = "kube_namespace_annotations" + descNamespaceAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." descNamespaceLabelsName = "kube_namespace_labels" descNamespaceLabelsHelp = "Kubernetes labels converted to Prometheus labels." descNamespaceLabelsDefaultLabels = []string{"namespace"} +) - namespaceMetricFamilies = []metric.FamilyGenerator{ - { - Name: "kube_namespace_created", - Type: metric.Gauge, - Help: "Unix creation timestamp", - GenerateFunc: wrapNamespaceFunc(func(n *v1.Namespace) *metric.Family { +func namespaceMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_namespace_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.STABLE, + "", + wrapNamespaceFunc(func(n *v1.Namespace) *metric.Family { ms := []*metric.Metric{} if !n.CreationTimestamp.IsZero() { ms = append(ms, &metric.Metric{ @@ -49,13 +60,34 @@ var ( Metrics: ms, } }), - }, - { - Name: descNamespaceLabelsName, - Type: metric.Gauge, - Help: descNamespaceLabelsHelp, - GenerateFunc: wrapNamespaceFunc(func(n *v1.Namespace) *metric.Family { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(n.Labels) + ), + *generator.NewFamilyGeneratorWithStability( + descNamespaceAnnotationsName, + descNamespaceAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapNamespaceFunc(func(n *v1.Namespace) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", n.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descNamespaceLabelsName, + descNamespaceLabelsHelp, + metric.Gauge, + basemetrics.STABLE, + "", + wrapNamespaceFunc(func(n *v1.Namespace) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", n.Labels, allowLabelsList) return &metric.Family{ Metrics: []*metric.Metric{ { @@ -66,12 +98,14 @@ var ( }, } }), - }, - { - Name: "kube_namespace_status_phase", - Type: metric.Gauge, - Help: "kubernetes namespace status phase.", - GenerateFunc: wrapNamespaceFunc(func(n *v1.Namespace) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_namespace_status_phase", + "kubernetes namespace status phase.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapNamespaceFunc(func(n *v1.Namespace) *metric.Family { ms := []*metric.Metric{ { LabelValues: []string{string(v1.NamespaceActive)}, @@ -91,12 +125,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_namespace_status_condition", - Type: metric.Gauge, - Help: "The condition of a namespace.", - GenerateFunc: wrapNamespaceFunc(func(n *v1.Namespace) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_namespace_status_condition", + "The condition of a namespace.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapNamespaceFunc(func(n *v1.Namespace) *metric.Family { ms := make([]*metric.Metric, len(n.Status.Conditions)*len(conditionStatuses)) for i, c := range n.Status.Conditions { conditionMetrics := addConditionMetrics(c.Status) @@ -115,9 +151,9 @@ var ( Metrics: ms, } }), - }, + ), } -) +} func wrapNamespaceFunc(f func(*v1.Namespace) *metric.Family) func(interface{}) *metric.Family { return func(obj interface{}) *metric.Family { @@ -126,21 +162,20 @@ func wrapNamespaceFunc(f func(*v1.Namespace) *metric.Family) func(interface{}) * metricFamily := f(namespace) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descNamespaceLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{namespace.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descNamespaceLabelsDefaultLabels, []string{namespace.Name}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createNamespaceListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createNamespaceListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.CoreV1().Namespaces().List(opts) + return kubeClient.CoreV1().Namespaces().List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.CoreV1().Namespaces().Watch(opts) + return kubeClient.CoreV1().Namespaces().Watch(context.TODO(), opts) }, } } diff --git a/internal/store/namespace_test.go b/internal/store/namespace_test.go index 97c74f9fb7..2f8161f094 100644 --- a/internal/store/namespace_test.go +++ b/internal/store/namespace_test.go @@ -23,18 +23,20 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) func TestNamespaceStore(t *testing.T) { // Fixed metadata on type and help text. We prepend this to every expected // output so we only have to modify a single place when doing adjustments. const metadata = ` - # HELP kube_namespace_created Unix creation timestamp + # HELP kube_namespace_annotations Kubernetes annotations converted to Prometheus labels. + # TYPE kube_namespace_annotations gauge + # HELP kube_namespace_created [STABLE] Unix creation timestamp # TYPE kube_namespace_created gauge - # HELP kube_namespace_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_namespace_labels [STABLE] Kubernetes labels converted to Prometheus labels. # TYPE kube_namespace_labels gauge - # HELP kube_namespace_status_phase kubernetes namespace status phase. + # HELP kube_namespace_status_phase [STABLE] kubernetes namespace status phase. # TYPE kube_namespace_status_phase gauge # HELP kube_namespace_status_condition The condition of a namespace. # TYPE kube_namespace_status_condition gauge @@ -54,6 +56,7 @@ func TestNamespaceStore(t *testing.T) { }, }, Want: metadata + ` + kube_namespace_annotations{namespace="nsActiveTest"} 1 kube_namespace_labels{namespace="nsActiveTest"} 1 kube_namespace_status_phase{namespace="nsActiveTest",phase="Active"} 1 kube_namespace_status_phase{namespace="nsActiveTest",phase="Terminating"} 0 @@ -72,6 +75,7 @@ func TestNamespaceStore(t *testing.T) { }, }, Want: metadata + ` + kube_namespace_annotations{namespace="nsTerminateTest"} 1 kube_namespace_labels{namespace="nsTerminateTest"} 1 kube_namespace_status_phase{namespace="nsTerminateTest",phase="Active"} 0 kube_namespace_status_phase{namespace="nsTerminateTest",phase="Terminating"} 1 @@ -95,6 +99,7 @@ func TestNamespaceStore(t *testing.T) { }, }, Want: metadata + ` + kube_namespace_annotations{namespace="nsTerminateWithConditionTest"} 1 kube_namespace_labels{namespace="nsTerminateWithConditionTest"} 1 kube_namespace_status_phase{namespace="nsTerminateWithConditionTest",phase="Active"} 0 kube_namespace_status_phase{namespace="nsTerminateWithConditionTest",phase="Terminating"} 1 @@ -127,8 +132,9 @@ func TestNamespaceStore(t *testing.T) { }, }, Want: metadata + ` + kube_namespace_annotations{namespace="ns1"} 1 kube_namespace_created{namespace="ns1"} 1.5e+09 - kube_namespace_labels{label_app="example1",namespace="ns1"} 1 + kube_namespace_labels{namespace="ns1"} 1 kube_namespace_status_phase{namespace="ns1",phase="Active"} 1 kube_namespace_status_phase{namespace="ns1",phase="Terminating"} 0 `, @@ -150,7 +156,8 @@ func TestNamespaceStore(t *testing.T) { }, }, Want: metadata + ` - kube_namespace_labels{label_app="example2",label_l2="label2",namespace="ns2"} 1 + kube_namespace_annotations{namespace="ns2"} 1 + kube_namespace_labels{namespace="ns2"} 1 kube_namespace_status_phase{namespace="ns2",phase="Active"} 1 kube_namespace_status_phase{namespace="ns2",phase="Terminating"} 0 `, @@ -158,8 +165,8 @@ func TestNamespaceStore(t *testing.T) { } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(namespaceMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(namespaceMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(namespaceMetricFamilies(nil, nil)) + c.Headers = generator.ExtractMetricFamilyHeaders(namespaceMetricFamilies(nil, nil)) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/networkpolicy.go b/internal/store/networkpolicy.go index 179af9b949..62eb87ee2e 100644 --- a/internal/store/networkpolicy.go +++ b/internal/store/networkpolicy.go @@ -17,25 +17,37 @@ limitations under the License. package store import ( + "context" + networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" - "k8s.io/kube-state-metrics/pkg/metric" + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) var ( + descNetworkPolicyAnnotationsName = "kube_networkpolicy_annotations" + descNetworkPolicyAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." + descNetworkPolicyLabelsName = "kube_networkpolicy_labels" + descNetworkPolicyLabelsHelp = "Kubernetes labels converted to Prometheus labels." descNetworkPolicyLabelsDefaultLabels = []string{"namespace", "networkpolicy"} +) - networkpolicyMetricFamilies = []metric.FamilyGenerator{ - { - Name: "kube_networkpolicy_created", - Type: metric.Gauge, - Help: "Unix creation timestamp of network policy", - GenerateFunc: wrapNetworkPolicyFunc(func(n *networkingv1.NetworkPolicy) *metric.Family { +func networkPolicyMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_networkpolicy_created", + "Unix creation timestamp of network policy", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapNetworkPolicyFunc(func(n *networkingv1.NetworkPolicy) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -46,13 +58,34 @@ var ( }, } }), - }, - { - Name: "kube_networkpolicy_labels", - Type: metric.Gauge, - Help: "Kubernetes labels converted to Prometheus labels", - GenerateFunc: wrapNetworkPolicyFunc(func(n *networkingv1.NetworkPolicy) *metric.Family { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(n.Labels) + ), + *generator.NewFamilyGeneratorWithStability( + descNetworkPolicyAnnotationsName, + descNetworkPolicyAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapNetworkPolicyFunc(func(n *networkingv1.NetworkPolicy) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", n.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descNetworkPolicyLabelsName, + descNetworkPolicyLabelsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapNetworkPolicyFunc(func(n *networkingv1.NetworkPolicy) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", n.Labels, allowLabelsList) return &metric.Family{ Metrics: []*metric.Metric{ { @@ -63,12 +96,14 @@ var ( }, } }), - }, - { - Name: "kube_networkpolicy_spec_ingress_rules", - Type: metric.Gauge, - Help: "Number of ingress rules on the networkpolicy", - GenerateFunc: wrapNetworkPolicyFunc(func(n *networkingv1.NetworkPolicy) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_networkpolicy_spec_ingress_rules", + "Number of ingress rules on the networkpolicy", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapNetworkPolicyFunc(func(n *networkingv1.NetworkPolicy) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -79,12 +114,14 @@ var ( }, } }), - }, - { - Name: "kube_networkpolicy_spec_egress_rules", - Type: metric.Gauge, - Help: "Number of egress rules on the networkpolicy", - GenerateFunc: wrapNetworkPolicyFunc(func(n *networkingv1.NetworkPolicy) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_networkpolicy_spec_egress_rules", + "Number of egress rules on the networkpolicy", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapNetworkPolicyFunc(func(n *networkingv1.NetworkPolicy) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -95,9 +132,9 @@ var ( }, } }), - }, + ), } -) +} func wrapNetworkPolicyFunc(f func(*networkingv1.NetworkPolicy) *metric.Family) func(interface{}) *metric.Family { return func(obj interface{}) *metric.Family { @@ -106,21 +143,22 @@ func wrapNetworkPolicyFunc(f func(*networkingv1.NetworkPolicy) *metric.Family) f metricFamily := f(networkPolicy) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descNetworkPolicyLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{networkPolicy.Namespace, networkPolicy.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descNetworkPolicyLabelsDefaultLabels, []string{networkPolicy.Namespace, networkPolicy.Name}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createNetworkPolicyListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createNetworkPolicyListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.NetworkingV1().NetworkPolicies(ns).List(opts) + opts.FieldSelector = fieldSelector + return kubeClient.NetworkingV1().NetworkPolicies(ns).List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.NetworkingV1().NetworkPolicies(ns).Watch(opts) + opts.FieldSelector = fieldSelector + return kubeClient.NetworkingV1().NetworkPolicies(ns).Watch(context.TODO(), opts) }, } } diff --git a/internal/store/networkpolicy_test.go b/internal/store/networkpolicy_test.go index 34bb19dc8c..22b693c081 100644 --- a/internal/store/networkpolicy_test.go +++ b/internal/store/networkpolicy_test.go @@ -22,7 +22,7 @@ import ( networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) func TestNetworkPolicyStore(t *testing.T) { @@ -68,7 +68,7 @@ func TestNetworkPolicyStore(t *testing.T) { }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(networkpolicyMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(networkPolicyMetricFamilies(nil, nil)) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %dth run:\n%s", i, err) } diff --git a/internal/store/node.go b/internal/store/node.go index e362a104ad..d4f80cb6b5 100644 --- a/internal/store/node.go +++ b/internal/store/node.go @@ -17,10 +17,14 @@ limitations under the License. package store import ( + "context" "strings" - "k8s.io/kube-state-metrics/pkg/constant" - "k8s.io/kube-state-metrics/pkg/metric" + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/constant" + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -31,228 +35,297 @@ import ( ) var ( + descNodeAnnotationsName = "kube_node_annotations" + descNodeAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." descNodeLabelsName = "kube_node_labels" descNodeLabelsHelp = "Kubernetes labels converted to Prometheus labels." descNodeLabelsDefaultLabels = []string{"node"} +) - nodeMetricFamilies = []metric.FamilyGenerator{ - { - Name: "kube_node_info", - Type: metric.Gauge, - Help: "Information about a cluster node.", - GenerateFunc: wrapNodeFunc(func(n *v1.Node) *metric.Family { - return &metric.Family{ - Metrics: []*metric.Metric{ - { - LabelKeys: []string{ - "kernel_version", - "os_image", - "container_runtime_version", - "kubelet_version", - "kubeproxy_version", - "provider_id", - "pod_cidr", - }, - LabelValues: []string{ - n.Status.NodeInfo.KernelVersion, - n.Status.NodeInfo.OSImage, - n.Status.NodeInfo.ContainerRuntimeVersion, - n.Status.NodeInfo.KubeletVersion, - n.Status.NodeInfo.KubeProxyVersion, - n.Spec.ProviderID, - n.Spec.PodCIDR, - }, - Value: 1, - }, - }, - } - }), - }, - { - Name: "kube_node_created", - Type: metric.Gauge, - Help: "Unix creation timestamp", - GenerateFunc: wrapNodeFunc(func(n *v1.Node) *metric.Family { - ms := []*metric.Metric{} - - if !n.CreationTimestamp.IsZero() { - ms = append(ms, &metric.Metric{ - - Value: float64(n.CreationTimestamp.Unix()), - }) - } - - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: descNodeLabelsName, - Type: metric.Gauge, - Help: descNodeLabelsHelp, - GenerateFunc: wrapNodeFunc(func(n *v1.Node) *metric.Family { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(n.Labels) - return &metric.Family{ - Metrics: []*metric.Metric{ - { - LabelKeys: labelKeys, - LabelValues: labelValues, - Value: 1, - }, - }, - } - }), - }, - { - Name: "kube_node_role", - Type: metric.Gauge, - Help: "The role of a cluster node.", - GenerateFunc: wrapNodeFunc(func(n *v1.Node) *metric.Family { - const prefix = "node-role.kubernetes.io/" - ms := []*metric.Metric{} - for lbl := range n.Labels { - if strings.HasPrefix(lbl, prefix) { - ms = append(ms, &metric.Metric{ - LabelKeys: []string{"role"}, - LabelValues: []string{strings.TrimPrefix(lbl, prefix)}, - Value: float64(1), - }) - } - } - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_node_spec_unschedulable", - Type: metric.Gauge, - Help: "Whether a node can schedule new pods.", - GenerateFunc: wrapNodeFunc(func(n *v1.Node) *metric.Family { - return &metric.Family{ - Metrics: []*metric.Metric{ - { - Value: boolFloat64(n.Spec.Unschedulable), - }, - }, - } - }), - }, - { - Name: "kube_node_spec_taint", - Type: metric.Gauge, - Help: "The taint of a cluster node.", - GenerateFunc: wrapNodeFunc(func(n *v1.Node) *metric.Family { - ms := make([]*metric.Metric, len(n.Spec.Taints)) - - for i, taint := range n.Spec.Taints { - // Taints are applied to repel pods from nodes that do not have a corresponding - // toleration. Many node conditions are optionally reflected as taints - // by the node controller in order to simplify scheduling constraints. - ms[i] = &metric.Metric{ - LabelKeys: []string{"key", "value", "effect"}, - LabelValues: []string{taint.Key, taint.Value, string(taint.Effect)}, - Value: 1, - } - } +func nodeMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + createNodeAnnotationsGenerator(allowAnnotationsList), + createNodeCreatedFamilyGenerator(), + createNodeDeletionTimestampFamilyGenerator(), + createNodeInfoFamilyGenerator(), + createNodeLabelsGenerator(allowLabelsList), + createNodeRoleFamilyGenerator(), + createNodeSpecTaintFamilyGenerator(), + createNodeSpecUnschedulableFamilyGenerator(), + createNodeStatusAllocatableFamilyGenerator(), + createNodeStatusCapacityFamilyGenerator(), + createNodeStatusConditionFamilyGenerator(), + } +} - return &metric.Family{ - Metrics: ms, - } - }), - }, - // This all-in-one metric family contains all conditions for extensibility. - // Third party plugin may report customized condition for cluster node - // (e.g. node-problem-detector), and Kubernetes may add new core - // conditions in future. - { - Name: "kube_node_status_condition", - Type: metric.Gauge, - Help: "The condition of a cluster node.", - GenerateFunc: wrapNodeFunc(func(n *v1.Node) *metric.Family { - ms := make([]*metric.Metric, len(n.Status.Conditions)*len(conditionStatuses)) - - // Collect node conditions and while default to false. - for i, c := range n.Status.Conditions { - conditionMetrics := addConditionMetrics(c.Status) - - for j, m := range conditionMetrics { - metric := m - - metric.LabelKeys = []string{"condition", "status"} - metric.LabelValues = append([]string{string(c.Type)}, metric.LabelValues...) - - ms[i*len(conditionStatuses)+j] = metric - } - } +func createNodeDeletionTimestampFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_node_deletion_timestamp", + "Unix deletion timestamp", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapNodeFunc(func(n *v1.Node) *metric.Family { + var ms []*metric.Metric + + if n.DeletionTimestamp != nil && !n.DeletionTimestamp.IsZero() { + ms = append(ms, &metric.Metric{ + Value: float64(n.DeletionTimestamp.Unix()), + }) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_node_status_phase", - Type: metric.Gauge, - Help: "The phase the node is currently in.", - GenerateFunc: wrapNodeFunc(func(n *v1.Node) *metric.Family { - p := n.Status.Phase - - if p == "" { - return &metric.Family{ - Metrics: []*metric.Metric{}, - } - } +func createNodeCreatedFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_node_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.STABLE, + "", + wrapNodeFunc(func(n *v1.Node) *metric.Family { + ms := []*metric.Metric{} + + if !n.CreationTimestamp.IsZero() { + ms = append(ms, &metric.Metric{ + + Value: float64(n.CreationTimestamp.Unix()), + }) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} - // Set current phase to 1, others to 0 if it is set. - ms := []*metric.Metric{ +func createNodeInfoFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_node_info", + "Information about a cluster node.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapNodeFunc(func(n *v1.Node) *metric.Family { + labelKeys := []string{ + "kernel_version", + "os_image", + "container_runtime_version", + "kubelet_version", + "kubeproxy_version", + "provider_id", + "pod_cidr", + "system_uuid", + } + labelValues := []string{ + n.Status.NodeInfo.KernelVersion, + n.Status.NodeInfo.OSImage, + n.Status.NodeInfo.ContainerRuntimeVersion, + n.Status.NodeInfo.KubeletVersion, + n.Status.NodeInfo.KubeProxyVersion, + n.Spec.ProviderID, + n.Spec.PodCIDR, + n.Status.NodeInfo.SystemUUID, + } + + internalIP := "" + for _, address := range n.Status.Addresses { + if address.Type == "InternalIP" { + internalIP = address.Address + } + } + labelKeys = append(labelKeys, "internal_ip") + labelValues = append(labelValues, internalIP) + + return &metric.Family{ + Metrics: []*metric.Metric{ { - LabelValues: []string{string(v1.NodePending)}, - Value: boolFloat64(p == v1.NodePending), + LabelKeys: labelKeys, + LabelValues: labelValues, + Value: 1, }, + }, + } + }), + ) +} + +func createNodeAnnotationsGenerator(allowAnnotationsList []string) generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + descNodeAnnotationsName, + descNodeAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapNodeFunc(func(n *v1.Node) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", n.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ { - LabelValues: []string{string(v1.NodeRunning)}, - Value: boolFloat64(p == v1.NodeRunning), + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, }, + }, + } + }), + ) +} + +func createNodeLabelsGenerator(allowLabelsList []string) generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + descNodeLabelsName, + descNodeLabelsHelp, + metric.Gauge, + basemetrics.STABLE, + "", + wrapNodeFunc(func(n *v1.Node) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", n.Labels, allowLabelsList) + return &metric.Family{ + Metrics: []*metric.Metric{ { - LabelValues: []string{string(v1.NodeTerminated)}, - Value: boolFloat64(p == v1.NodeTerminated), + LabelKeys: labelKeys, + LabelValues: labelValues, + Value: 1, }, - } + }, + } + }), + ) +} - for _, metric := range ms { - metric.LabelKeys = []string{"phase"} +func createNodeRoleFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_node_role", + "The role of a cluster node.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapNodeFunc(func(n *v1.Node) *metric.Family { + const prefix = "node-role.kubernetes.io/" + ms := []*metric.Metric{} + for lbl := range n.Labels { + if strings.HasPrefix(lbl, prefix) { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"role"}, + LabelValues: []string{strings.TrimPrefix(lbl, prefix)}, + Value: float64(1), + }) } + } + return &metric.Family{ + Metrics: ms, + } + }), + ) +} - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_node_status_capacity", - Type: metric.Gauge, - Help: "The capacity for different resources of a node.", - GenerateFunc: wrapNodeFunc(func(n *v1.Node) *metric.Family { - ms := []*metric.Metric{} - - capacity := n.Status.Capacity - for resourceName, val := range capacity { - switch resourceName { - case v1.ResourceCPU: +func createNodeSpecTaintFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_node_spec_taint", + "The taint of a cluster node.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapNodeFunc(func(n *v1.Node) *metric.Family { + ms := make([]*metric.Metric, len(n.Spec.Taints)) + + for i, taint := range n.Spec.Taints { + // Taints are applied to repel pods from nodes that do not have a corresponding + // toleration. Many node conditions are optionally reflected as taints + // by the node controller in order to simplify scheduling constraints. + ms[i] = &metric.Metric{ + LabelKeys: []string{"key", "value", "effect"}, + LabelValues: []string{taint.Key, taint.Value, string(taint.Effect)}, + Value: 1, + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createNodeSpecUnschedulableFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_node_spec_unschedulable", + "Whether a node can schedule new pods.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapNodeFunc(func(n *v1.Node) *metric.Family { + return &metric.Family{ + Metrics: []*metric.Metric{ + { + Value: boolFloat64(n.Spec.Unschedulable), + }, + }, + } + }), + ) +} + +func createNodeStatusAllocatableFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_node_status_allocatable", + "The allocatable for different resources of a node that are available for scheduling.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapNodeFunc(func(n *v1.Node) *metric.Family { + ms := []*metric.Metric{} + + allocatable := n.Status.Allocatable + + for resourceName, val := range allocatable { + switch resourceName { + case v1.ResourceCPU: + ms = append(ms, &metric.Metric{ + LabelValues: []string{ + sanitizeLabelName(string(resourceName)), + string(constant.UnitCore), + }, + Value: float64(val.MilliValue()) / 1000, + }) + case v1.ResourceStorage: + fallthrough + case v1.ResourceEphemeralStorage: + fallthrough + case v1.ResourceMemory: + ms = append(ms, &metric.Metric{ + LabelValues: []string{ + sanitizeLabelName(string(resourceName)), + string(constant.UnitByte), + }, + Value: float64(val.MilliValue()) / 1000, + }) + case v1.ResourcePods: + ms = append(ms, &metric.Metric{ + LabelValues: []string{ + sanitizeLabelName(string(resourceName)), + string(constant.UnitInteger), + }, + Value: float64(val.MilliValue()) / 1000, + }) + default: + if isHugePageResourceName(resourceName) { ms = append(ms, &metric.Metric{ LabelValues: []string{ sanitizeLabelName(string(resourceName)), - string(constant.UnitCore), + string(constant.UnitByte), }, Value: float64(val.MilliValue()) / 1000, }) - case v1.ResourceStorage: - fallthrough - case v1.ResourceEphemeralStorage: - fallthrough - case v1.ResourceMemory: + } + if isAttachableVolumeResourceName(resourceName) { ms = append(ms, &metric.Metric{ LabelValues: []string{ sanitizeLabelName(string(resourceName)), @@ -260,7 +333,8 @@ var ( }, Value: float64(val.MilliValue()) / 1000, }) - case v1.ResourcePods: + } + if isExtendedResourceName(resourceName) { ms = append(ms, &metric.Metric{ LabelValues: []string{ sanitizeLabelName(string(resourceName)), @@ -268,128 +342,73 @@ var ( }, Value: float64(val.MilliValue()) / 1000, }) - default: - if isHugePageResourceName(resourceName) { - ms = append(ms, &metric.Metric{ - LabelValues: []string{ - sanitizeLabelName(string(resourceName)), - string(constant.UnitByte), - }, - Value: float64(val.MilliValue()) / 1000, - }) - } - if isAttachableVolumeResourceName(resourceName) { - ms = append(ms, &metric.Metric{ - LabelValues: []string{ - sanitizeLabelName(string(resourceName)), - string(constant.UnitByte), - }, - Value: float64(val.MilliValue()) / 1000, - }) - } - if isExtendedResourceName(resourceName) { - ms = append(ms, &metric.Metric{ - LabelValues: []string{ - sanitizeLabelName(string(resourceName)), - string(constant.UnitInteger), - }, - Value: float64(val.MilliValue()) / 1000, - }) - } } } + } - for _, metric := range ms { - metric.LabelKeys = []string{"resource", "unit"} - } + for _, m := range ms { + m.LabelKeys = []string{"resource", "unit"} + } - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_node_status_capacity_pods", - Type: metric.Gauge, - Help: "The total pod resources of the node.", - GenerateFunc: wrapNodeFunc(func(n *v1.Node) *metric.Family { - ms := []*metric.Metric{} - - // Add capacity and allocatable resources if they are set. - if v, ok := n.Status.Capacity[v1.ResourcePods]; ok { - ms = append(ms, &metric.Metric{ + return &metric.Family{ + Metrics: ms, + } + }), + ) +} - Value: float64(v.MilliValue()) / 1000, +func createNodeStatusCapacityFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_node_status_capacity", + "The capacity for different resources of a node.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapNodeFunc(func(n *v1.Node) *metric.Family { + ms := []*metric.Metric{} + + capacity := n.Status.Capacity + for resourceName, val := range capacity { + switch resourceName { + case v1.ResourceCPU: + ms = append(ms, &metric.Metric{ + LabelValues: []string{ + sanitizeLabelName(string(resourceName)), + string(constant.UnitCore), + }, + Value: float64(val.MilliValue()) / 1000, }) - } - - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_node_status_capacity_cpu_cores", - Type: metric.Gauge, - Help: "The total CPU resources of the node.", - GenerateFunc: wrapNodeFunc(func(n *v1.Node) *metric.Family { - ms := []*metric.Metric{} - - // Add capacity and allocatable resources if they are set. - if v, ok := n.Status.Capacity[v1.ResourceCPU]; ok { + case v1.ResourceStorage: + fallthrough + case v1.ResourceEphemeralStorage: + fallthrough + case v1.ResourceMemory: ms = append(ms, &metric.Metric{ - Value: float64(v.MilliValue()) / 1000, + LabelValues: []string{ + sanitizeLabelName(string(resourceName)), + string(constant.UnitByte), + }, + Value: float64(val.MilliValue()) / 1000, }) - } - - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_node_status_capacity_memory_bytes", - Type: metric.Gauge, - Help: "The total memory resources of the node.", - GenerateFunc: wrapNodeFunc(func(n *v1.Node) *metric.Family { - ms := []*metric.Metric{} - - // Add capacity and allocatable resources if they are set. - if v, ok := n.Status.Capacity[v1.ResourceMemory]; ok { + case v1.ResourcePods: ms = append(ms, &metric.Metric{ - Value: float64(v.MilliValue()) / 1000, + LabelValues: []string{ + sanitizeLabelName(string(resourceName)), + string(constant.UnitInteger), + }, + Value: float64(val.MilliValue()) / 1000, }) - } - - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_node_status_allocatable", - Type: metric.Gauge, - Help: "The allocatable for different resources of a node that are available for scheduling.", - GenerateFunc: wrapNodeFunc(func(n *v1.Node) *metric.Family { - ms := []*metric.Metric{} - - allocatable := n.Status.Allocatable - - for resourceName, val := range allocatable { - switch resourceName { - case v1.ResourceCPU: + default: + if isHugePageResourceName(resourceName) { ms = append(ms, &metric.Metric{ LabelValues: []string{ sanitizeLabelName(string(resourceName)), - string(constant.UnitCore), + string(constant.UnitByte), }, Value: float64(val.MilliValue()) / 1000, }) - case v1.ResourceStorage: - fallthrough - case v1.ResourceEphemeralStorage: - fallthrough - case v1.ResourceMemory: + } + if isAttachableVolumeResourceName(resourceName) { ms = append(ms, &metric.Metric{ LabelValues: []string{ sanitizeLabelName(string(resourceName)), @@ -397,7 +416,8 @@ var ( }, Value: float64(val.MilliValue()) / 1000, }) - case v1.ResourcePods: + } + if isExtendedResourceName(resourceName) { ms = append(ms, &metric.Metric{ LabelValues: []string{ sanitizeLabelName(string(resourceName)), @@ -405,106 +425,55 @@ var ( }, Value: float64(val.MilliValue()) / 1000, }) - default: - if isHugePageResourceName(resourceName) { - ms = append(ms, &metric.Metric{ - LabelValues: []string{ - sanitizeLabelName(string(resourceName)), - string(constant.UnitByte), - }, - Value: float64(val.MilliValue()) / 1000, - }) - } - if isAttachableVolumeResourceName(resourceName) { - ms = append(ms, &metric.Metric{ - LabelValues: []string{ - sanitizeLabelName(string(resourceName)), - string(constant.UnitByte), - }, - Value: float64(val.MilliValue()) / 1000, - }) - } - if isExtendedResourceName(resourceName) { - ms = append(ms, &metric.Metric{ - LabelValues: []string{ - sanitizeLabelName(string(resourceName)), - string(constant.UnitInteger), - }, - Value: float64(val.MilliValue()) / 1000, - }) - } } } + } - for _, m := range ms { - m.LabelKeys = []string{"resource", "unit"} - } - - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_node_status_allocatable_pods", - Type: metric.Gauge, - Help: "The pod resources of a node that are available for scheduling.", - GenerateFunc: wrapNodeFunc(func(n *v1.Node) *metric.Family { - ms := []*metric.Metric{} - - // Add capacity and allocatable resources if they are set. - if v, ok := n.Status.Allocatable[v1.ResourcePods]; ok { - ms = append(ms, &metric.Metric{ - Value: float64(v.MilliValue()) / 1000, - }) - } + for _, metric := range ms { + metric.LabelKeys = []string{"resource", "unit"} + } - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_node_status_allocatable_cpu_cores", - Type: metric.Gauge, - Help: "The CPU resources of a node that are available for scheduling.", - GenerateFunc: wrapNodeFunc(func(n *v1.Node) *metric.Family { - ms := []*metric.Metric{} - - // Add capacity and allocatable resources if they are set. - if v, ok := n.Status.Allocatable[v1.ResourceCPU]; ok { - ms = append(ms, &metric.Metric{ - Value: float64(v.MilliValue()) / 1000, - }) - } - - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_node_status_allocatable_memory_bytes", - Type: metric.Gauge, - Help: "The memory resources of a node that are available for scheduling.", - GenerateFunc: wrapNodeFunc(func(n *v1.Node) *metric.Family { - ms := []*metric.Metric{} - - // Add capacity and allocatable resources if they are set. - if v, ok := n.Status.Allocatable[v1.ResourceMemory]; ok { - ms = append(ms, &metric.Metric{ - - Value: float64(v.MilliValue()) / 1000, - }) - } + return &metric.Family{ + Metrics: ms, + } + }), + ) +} - return &metric.Family{ - Metrics: ms, - } - }), - }, - } -) +// createNodeStatusConditionFamilyGenerator returns an all-in-one metric family +// containing all conditions for extensibility. Third party plugin may report +// customized condition for cluster node (e.g. node-problem-detector), and +// Kubernetes may add new core conditions in future. +func createNodeStatusConditionFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_node_status_condition", + "The condition of a cluster node.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapNodeFunc(func(n *v1.Node) *metric.Family { + ms := make([]*metric.Metric, len(n.Status.Conditions)*len(conditionStatuses)) + + // Collect node conditions and while default to false. + for i, c := range n.Status.Conditions { + conditionMetrics := addConditionMetrics(c.Status) + + for j, m := range conditionMetrics { + metric := m + + metric.LabelKeys = []string{"condition", "status"} + metric.LabelValues = append([]string{string(c.Type)}, metric.LabelValues...) + + ms[i*len(conditionStatuses)+j] = metric + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} func wrapNodeFunc(f func(*v1.Node) *metric.Family) func(interface{}) *metric.Family { return func(obj interface{}) *metric.Family { @@ -513,21 +482,20 @@ func wrapNodeFunc(f func(*v1.Node) *metric.Family) func(interface{}) *metric.Fam metricFamily := f(node) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descNodeLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{node.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descNodeLabelsDefaultLabels, []string{node.Name}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createNodeListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createNodeListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.CoreV1().Nodes().List(opts) + return kubeClient.CoreV1().Nodes().List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.CoreV1().Nodes().Watch(opts) + return kubeClient.CoreV1().Nodes().Watch(context.TODO(), opts) }, } } diff --git a/internal/store/node_test.go b/internal/store/node_test.go index 888471e3d5..0e44431705 100644 --- a/internal/store/node_test.go +++ b/internal/store/node_test.go @@ -24,7 +24,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) func TestNodeStore(t *testing.T) { @@ -42,6 +42,10 @@ func TestNodeStore(t *testing.T) { KubeProxyVersion: "kubeproxy", OSImage: "osimage", ContainerRuntimeVersion: "rkt", + SystemUUID: "6a934e21-5207-4a84-baea-3a952d926c80", + }, + Addresses: []v1.NodeAddress{ + {Type: "InternalIP", Address: "1.2.3.4"}, }, }, Spec: v1.NodeSpec{ @@ -50,13 +54,13 @@ func TestNodeStore(t *testing.T) { }, }, Want: ` - # HELP kube_node_info Information about a cluster node. - # HELP kube_node_labels Kubernetes labels converted to Prometheus labels. - # HELP kube_node_spec_unschedulable Whether a node can schedule new pods. + # HELP kube_node_info [STABLE] Information about a cluster node. + # HELP kube_node_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # HELP kube_node_spec_unschedulable [STABLE] Whether a node can schedule new pods. # TYPE kube_node_info gauge # TYPE kube_node_labels gauge # TYPE kube_node_spec_unschedulable gauge - kube_node_info{container_runtime_version="rkt",kernel_version="kernel",kubelet_version="kubelet",kubeproxy_version="kubeproxy",node="127.0.0.1",os_image="osimage",pod_cidr="172.24.10.0/24",provider_id="provider://i-uniqueid"} 1 + kube_node_info{container_runtime_version="rkt",kernel_version="kernel",kubelet_version="kubelet",kubeproxy_version="kubeproxy",node="127.0.0.1",os_image="osimage",pod_cidr="172.24.10.0/24",provider_id="provider://i-uniqueid",internal_ip="1.2.3.4",system_uuid="6a934e21-5207-4a84-baea-3a952d926c80"} 1 kube_node_labels{node="127.0.0.1"} 1 kube_node_spec_unschedulable{node="127.0.0.1"} 0 `, @@ -70,9 +74,9 @@ func TestNodeStore(t *testing.T) { Spec: v1.NodeSpec{}, }, Want: ` - # HELP kube_node_info Information about a cluster node. + # HELP kube_node_info [STABLE] Information about a cluster node. # TYPE kube_node_info gauge - kube_node_info{container_runtime_version="",kernel_version="",kubelet_version="",kubeproxy_version="",node="",os_image="",pod_cidr="",provider_id=""} 1 + kube_node_info{container_runtime_version="",kernel_version="",kubelet_version="",kubeproxy_version="",node="",os_image="",pod_cidr="",provider_id="",internal_ip="",system_uuid=""} 1 `, MetricNames: []string{"kube_node_info"}, }, @@ -98,6 +102,10 @@ func TestNodeStore(t *testing.T) { KubeProxyVersion: "kubeproxy", OSImage: "osimage", ContainerRuntimeVersion: "rkt", + SystemUUID: "6a934e21-5207-4a84-baea-3a952d926c80", + }, + Addresses: []v1.NodeAddress{ + {Type: "InternalIP", Address: "1.2.3.4"}, }, Capacity: v1.ResourceList{ v1.ResourceCPU: resource.MustParse("4.3"), @@ -118,49 +126,31 @@ func TestNodeStore(t *testing.T) { }, }, Want: ` - # HELP kube_node_created Unix creation timestamp - # HELP kube_node_info Information about a cluster node. - # HELP kube_node_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_node_created [STABLE] Unix creation timestamp + # HELP kube_node_info [STABLE] Information about a cluster node. + # HELP kube_node_labels [STABLE] Kubernetes labels converted to Prometheus labels. # HELP kube_node_role The role of a cluster node. - # HELP kube_node_spec_unschedulable Whether a node can schedule new pods. - # HELP kube_node_status_allocatable The allocatable for different resources of a node that are available for scheduling. - # HELP kube_node_status_allocatable_cpu_cores The CPU resources of a node that are available for scheduling. - # HELP kube_node_status_allocatable_memory_bytes The memory resources of a node that are available for scheduling. - # HELP kube_node_status_allocatable_pods The pod resources of a node that are available for scheduling. - # HELP kube_node_status_capacity The capacity for different resources of a node. - # HELP kube_node_status_capacity_cpu_cores The total CPU resources of the node. - # HELP kube_node_status_capacity_memory_bytes The total memory resources of the node. - # HELP kube_node_status_capacity_pods The total pod resources of the node. + # HELP kube_node_spec_unschedulable [STABLE] Whether a node can schedule new pods. + # HELP kube_node_status_allocatable [STABLE] The allocatable for different resources of a node that are available for scheduling. + # HELP kube_node_status_capacity [STABLE] The capacity for different resources of a node. # TYPE kube_node_created gauge # TYPE kube_node_info gauge # TYPE kube_node_labels gauge # TYPE kube_node_role gauge # TYPE kube_node_spec_unschedulable gauge # TYPE kube_node_status_allocatable gauge - # TYPE kube_node_status_allocatable_cpu_cores gauge - # TYPE kube_node_status_allocatable_memory_bytes gauge - # TYPE kube_node_status_allocatable_pods gauge # TYPE kube_node_status_capacity gauge - # TYPE kube_node_status_capacity_cpu_cores gauge - # TYPE kube_node_status_capacity_memory_bytes gauge - # TYPE kube_node_status_capacity_pods gauge kube_node_created{node="127.0.0.1"} 1.5e+09 - kube_node_info{container_runtime_version="rkt",kernel_version="kernel",kubelet_version="kubelet",kubeproxy_version="kubeproxy",node="127.0.0.1",os_image="osimage",pod_cidr="172.24.10.0/24",provider_id="provider://i-randomidentifier"} 1 - kube_node_labels{label_node_role_kubernetes_io_master="",node="127.0.0.1"} 1 + kube_node_info{container_runtime_version="rkt",kernel_version="kernel",kubelet_version="kubelet",kubeproxy_version="kubeproxy",node="127.0.0.1",os_image="osimage",pod_cidr="172.24.10.0/24",provider_id="provider://i-randomidentifier",internal_ip="1.2.3.4",system_uuid="6a934e21-5207-4a84-baea-3a952d926c80"} 1 + kube_node_labels{node="127.0.0.1"} 1 kube_node_role{node="127.0.0.1",role="master"} 1 kube_node_spec_unschedulable{node="127.0.0.1"} 1 - kube_node_status_allocatable_cpu_cores{node="127.0.0.1"} 3 - kube_node_status_allocatable_memory_bytes{node="127.0.0.1"} 1e+09 - kube_node_status_allocatable_pods{node="127.0.0.1"} 555 kube_node_status_allocatable{node="127.0.0.1",resource="cpu",unit="core"} 3 kube_node_status_allocatable{node="127.0.0.1",resource="ephemeral_storage",unit="byte"} 3e+09 kube_node_status_allocatable{node="127.0.0.1",resource="memory",unit="byte"} 1e+09 kube_node_status_allocatable{node="127.0.0.1",resource="nvidia_com_gpu",unit="integer"} 1 kube_node_status_allocatable{node="127.0.0.1",resource="pods",unit="integer"} 555 kube_node_status_allocatable{node="127.0.0.1",resource="storage",unit="byte"} 2e+09 - kube_node_status_capacity_cpu_cores{node="127.0.0.1"} 4.3 - kube_node_status_capacity_memory_bytes{node="127.0.0.1"} 2e+09 - kube_node_status_capacity_pods{node="127.0.0.1"} 1000 kube_node_status_capacity{node="127.0.0.1",resource="cpu",unit="core"} 4.3 kube_node_status_capacity{node="127.0.0.1",resource="ephemeral_storage",unit="byte"} 4e+09 kube_node_status_capacity{node="127.0.0.1",resource="memory",unit="byte"} 2e+09 @@ -170,13 +160,7 @@ func TestNodeStore(t *testing.T) { `, MetricNames: []string{ "kube_node_status_capacity", - "kube_node_status_capacity_pods", - "kube_node_status_capacity_memory_bytes", - "kube_node_status_capacity_cpu_cores", "kube_node_status_allocatable", - "kube_node_status_allocatable_pods", - "kube_node_status_allocatable_memory_bytes", - "kube_node_status_allocatable_cpu_cores", "kube_node_spec_unschedulable", "kube_node_labels", "kube_node_role", @@ -184,61 +168,6 @@ func TestNodeStore(t *testing.T) { "kube_node_created", }, }, - // Verify phase enumerations. - { - Obj: &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "127.0.0.1", - }, - Status: v1.NodeStatus{ - Phase: v1.NodeRunning, - }, - }, - Want: ` - # HELP kube_node_status_phase The phase the node is currently in. - # TYPE kube_node_status_phase gauge - kube_node_status_phase{node="127.0.0.1",phase="Terminated"} 0 - kube_node_status_phase{node="127.0.0.1",phase="Running"} 1 - kube_node_status_phase{node="127.0.0.1",phase="Pending"} 0 -`, - MetricNames: []string{"kube_node_status_phase"}, - }, - { - Obj: &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "127.0.0.2", - }, - Status: v1.NodeStatus{ - Phase: v1.NodePending, - }, - }, - Want: ` - # HELP kube_node_status_phase The phase the node is currently in. - # TYPE kube_node_status_phase gauge - kube_node_status_phase{node="127.0.0.2",phase="Terminated"} 0 - kube_node_status_phase{node="127.0.0.2",phase="Running"} 0 - kube_node_status_phase{node="127.0.0.2",phase="Pending"} 1 -`, - MetricNames: []string{"kube_node_status_phase"}, - }, - { - Obj: &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "127.0.0.3", - }, - Status: v1.NodeStatus{ - Phase: v1.NodeTerminated, - }, - }, - Want: ` - # HELP kube_node_status_phase The phase the node is currently in. - # TYPE kube_node_status_phase gauge - kube_node_status_phase{node="127.0.0.3",phase="Terminated"} 1 - kube_node_status_phase{node="127.0.0.3",phase="Running"} 0 - kube_node_status_phase{node="127.0.0.3",phase="Pending"} 0 -`, - MetricNames: []string{"kube_node_status_phase"}, - }, // Verify StatusCondition { Obj: &v1.Node{ @@ -254,7 +183,7 @@ func TestNodeStore(t *testing.T) { }, }, Want: ` - # HELP kube_node_status_condition The condition of a cluster node. + # HELP kube_node_status_condition [STABLE] The condition of a cluster node. # TYPE kube_node_status_condition gauge kube_node_status_condition{condition="CustomizedType",node="127.0.0.1",status="false"} 0 kube_node_status_condition{condition="CustomizedType",node="127.0.0.1",status="true"} 1 @@ -282,7 +211,7 @@ func TestNodeStore(t *testing.T) { }, }, Want: ` - # HELP kube_node_status_condition The condition of a cluster node. + # HELP kube_node_status_condition [STABLE] The condition of a cluster node. # TYPE kube_node_status_condition gauge kube_node_status_condition{condition="CustomizedType",node="127.0.0.2",status="false"} 0 kube_node_status_condition{condition="CustomizedType",node="127.0.0.2",status="true"} 0 @@ -310,7 +239,7 @@ func TestNodeStore(t *testing.T) { }, }, Want: ` - # HELP kube_node_status_condition The condition of a cluster node. + # HELP kube_node_status_condition [STABLE] The condition of a cluster node. # TYPE kube_node_status_condition gauge kube_node_status_condition{condition="CustomizedType",node="127.0.0.3",status="false"} 1 kube_node_status_condition{condition="CustomizedType",node="127.0.0.3",status="true"} 0 @@ -339,7 +268,7 @@ func TestNodeStore(t *testing.T) { }, }, Want: ` - # HELP kube_node_spec_taint The taint of a cluster node. + # HELP kube_node_spec_taint [STABLE] The taint of a cluster node. # TYPE kube_node_spec_taint gauge kube_node_spec_taint{effect="PreferNoSchedule",key="Dedicated",node="127.0.0.1",value=""} 1 kube_node_spec_taint{effect="PreferNoSchedule",key="Accelerated",node="127.0.0.1",value="gpu"} 1 @@ -349,8 +278,8 @@ func TestNodeStore(t *testing.T) { }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(nodeMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(nodeMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(nodeMetricFamilies(nil, nil)) + c.Headers = generator.ExtractMetricFamilyHeaders(nodeMetricFamilies(nil, nil)) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/persistentvolume.go b/internal/store/persistentvolume.go index 9737df0925..35b7ea071a 100644 --- a/internal/store/persistentvolume.go +++ b/internal/store/persistentvolume.go @@ -17,7 +17,13 @@ limitations under the License. package store import ( - "k8s.io/kube-state-metrics/pkg/metric" + "context" + "strconv" + + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -28,17 +34,77 @@ import ( ) var ( + descPersistentVolumeClaimRefName = "kube_persistentvolume_claim_ref" + descPersistentVolumeClaimRefHelp = "Information about the Persistent Volume Claim Reference." + descPersistentVolumeClaimRefDefaultLabels = []string{"persistentvolume"} + + descPersistentVolumeAnnotationsName = "kube_persistentvolume_annotations" + descPersistentVolumeAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." descPersistentVolumeLabelsName = "kube_persistentvolume_labels" descPersistentVolumeLabelsHelp = "Kubernetes labels converted to Prometheus labels." descPersistentVolumeLabelsDefaultLabels = []string{"persistentvolume"} +) + +func persistentVolumeMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + descPersistentVolumeClaimRefName, + descPersistentVolumeClaimRefHelp, + metric.Gauge, + basemetrics.STABLE, + "", + wrapPersistentVolumeFunc(func(p *v1.PersistentVolume) *metric.Family { + claimRef := p.Spec.ClaimRef - persistentVolumeMetricFamilies = []metric.FamilyGenerator{ - { - Name: descPersistentVolumeLabelsName, - Type: metric.Gauge, - Help: descPersistentVolumeLabelsHelp, - GenerateFunc: wrapPersistentVolumeFunc(func(p *v1.PersistentVolume) *metric.Family { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(p.Labels) + if claimRef == nil { + return &metric.Family{ + Metrics: []*metric.Metric{}, + } + } + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: []string{ + "name", + "claim_namespace", + }, + LabelValues: []string{ + p.Spec.ClaimRef.Name, + p.Spec.ClaimRef.Namespace, + }, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descPersistentVolumeAnnotationsName, + descPersistentVolumeAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPersistentVolumeFunc(func(p *v1.PersistentVolume) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", p.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descPersistentVolumeLabelsName, + descPersistentVolumeLabelsHelp, + metric.Gauge, + basemetrics.STABLE, + "", + wrapPersistentVolumeFunc(func(p *v1.PersistentVolume) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", p.Labels, allowLabelsList) return &metric.Family{ Metrics: []*metric.Metric{ { @@ -49,12 +115,14 @@ var ( }, } }), - }, - { - Name: "kube_persistentvolume_status_phase", - Type: metric.Gauge, - Help: "The phase indicates if a volume is available, bound to a claim, or released by a claim.", - GenerateFunc: wrapPersistentVolumeFunc(func(p *v1.PersistentVolume) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_persistentvolume_status_phase", + "The phase indicates if a volume is available, bound to a claim, or released by a claim.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPersistentVolumeFunc(func(p *v1.PersistentVolume) *metric.Family { phase := p.Status.Phase if phase == "" { @@ -95,28 +163,132 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_persistentvolume_info", - Type: metric.Gauge, - Help: "Information about persistentvolume.", - GenerateFunc: wrapPersistentVolumeFunc(func(p *v1.PersistentVolume) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_persistentvolume_info", + "Information about persistentvolume.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPersistentVolumeFunc(func(p *v1.PersistentVolume) *metric.Family { + var ( + gcePDDiskName, + ebsVolumeID, + azureDiskName, + fcWWIDs, fcLun, fcTargetWWNs, + iscsiTargetPortal, iscsiIQN, iscsiLun, iscsiInitiatorName, + nfsServer, nfsPath, + csiDriver, csiVolumeHandle, + localFS, localPath, + hostPath, hostPathType string + ) + + switch { + case p.Spec.PersistentVolumeSource.GCEPersistentDisk != nil: + gcePDDiskName = p.Spec.PersistentVolumeSource.GCEPersistentDisk.PDName + case p.Spec.PersistentVolumeSource.AWSElasticBlockStore != nil: + ebsVolumeID = p.Spec.PersistentVolumeSource.AWSElasticBlockStore.VolumeID + case p.Spec.PersistentVolumeSource.AzureDisk != nil: + azureDiskName = p.Spec.PersistentVolumeSource.AzureDisk.DiskName + case p.Spec.PersistentVolumeSource.FC != nil: + if p.Spec.PersistentVolumeSource.FC.Lun != nil { + fcLun = strconv.FormatInt(int64(*p.Spec.PersistentVolumeSource.FC.Lun), 10) + } + for _, wwn := range p.Spec.PersistentVolumeSource.FC.TargetWWNs { + if len(fcTargetWWNs) != 0 { + fcTargetWWNs += "," + } + fcTargetWWNs += wwn + } + for _, wwid := range p.Spec.PersistentVolumeSource.FC.WWIDs { + if len(fcWWIDs) != 0 { + fcWWIDs += "," + } + fcWWIDs += wwid + } + case p.Spec.PersistentVolumeSource.ISCSI != nil: + iscsiTargetPortal = p.Spec.PersistentVolumeSource.ISCSI.TargetPortal + iscsiIQN = p.Spec.PersistentVolumeSource.ISCSI.IQN + iscsiLun = strconv.FormatInt(int64(p.Spec.PersistentVolumeSource.ISCSI.Lun), 10) + if p.Spec.PersistentVolumeSource.ISCSI.InitiatorName != nil { + iscsiInitiatorName = *p.Spec.PersistentVolumeSource.ISCSI.InitiatorName + } + case p.Spec.PersistentVolumeSource.NFS != nil: + nfsServer = p.Spec.PersistentVolumeSource.NFS.Server + nfsPath = p.Spec.PersistentVolumeSource.NFS.Path + case p.Spec.PersistentVolumeSource.CSI != nil: + csiDriver = p.Spec.PersistentVolumeSource.CSI.Driver + csiVolumeHandle = p.Spec.PersistentVolumeSource.CSI.VolumeHandle + case p.Spec.PersistentVolumeSource.Local != nil: + localPath = p.Spec.PersistentVolumeSource.Local.Path + if p.Spec.PersistentVolumeSource.Local.FSType != nil { + localFS = *p.Spec.PersistentVolumeSource.Local.FSType + } + case p.Spec.PersistentVolumeSource.HostPath != nil: + hostPath = p.Spec.PersistentVolumeSource.HostPath.Path + if p.Spec.PersistentVolumeSource.HostPath.Type != nil { + hostPathType = string(*p.Spec.PersistentVolumeSource.HostPath.Type) + } + } + return &metric.Family{ Metrics: []*metric.Metric{ { - LabelKeys: []string{"storageclass"}, - LabelValues: []string{p.Spec.StorageClassName}, - Value: 1, + LabelKeys: []string{ + "storageclass", + "gce_persistent_disk_name", + "ebs_volume_id", + "azure_disk_name", + "fc_wwids", + "fc_lun", + "fc_target_wwns", + "iscsi_target_portal", + "iscsi_iqn", + "iscsi_lun", + "iscsi_initiator_name", + "nfs_server", + "nfs_path", + "csi_driver", + "csi_volume_handle", + "local_path", + "local_fs", + "host_path", + "host_path_type", + }, + LabelValues: []string{ + p.Spec.StorageClassName, + gcePDDiskName, + ebsVolumeID, + azureDiskName, + fcWWIDs, + fcLun, + fcTargetWWNs, + iscsiTargetPortal, + iscsiIQN, + iscsiLun, + iscsiInitiatorName, + nfsServer, + nfsPath, + csiDriver, + csiVolumeHandle, + localPath, + localFS, + hostPath, + hostPathType, + }, + Value: 1, }, }, } }), - }, - { - Name: "kube_persistentvolume_capacity_bytes", - Type: metric.Gauge, - Help: "Persistentvolume capacity in bytes.", - GenerateFunc: wrapPersistentVolumeFunc(func(p *v1.PersistentVolume) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_persistentvolume_capacity_bytes", + "Persistentvolume capacity in bytes.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPersistentVolumeFunc(func(p *v1.PersistentVolume) *metric.Family { storage := p.Spec.Capacity[v1.ResourceStorage] return &metric.Family{ Metrics: []*metric.Metric{ @@ -126,9 +298,31 @@ var ( }, } }), - }, + ), + *generator.NewFamilyGeneratorWithStability( + "kube_persistentvolume_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPersistentVolumeFunc(func(p *v1.PersistentVolume) *metric.Family { + ms := []*metric.Metric{} + + if !p.CreationTimestamp.IsZero() { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64(p.CreationTimestamp.Unix()), + }) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ), } -) +} func wrapPersistentVolumeFunc(f func(*v1.PersistentVolume) *metric.Family) func(interface{}) *metric.Family { return func(obj interface{}) *metric.Family { @@ -137,21 +331,20 @@ func wrapPersistentVolumeFunc(f func(*v1.PersistentVolume) *metric.Family) func( metricFamily := f(persistentVolume) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descPersistentVolumeLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{persistentVolume.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descPersistentVolumeLabelsDefaultLabels, []string{persistentVolume.Name}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createPersistentVolumeListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createPersistentVolumeListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.CoreV1().PersistentVolumes().List(opts) + return kubeClient.CoreV1().PersistentVolumes().List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.CoreV1().PersistentVolumes().Watch(opts) + return kubeClient.CoreV1().PersistentVolumes().Watch(context.TODO(), opts) }, } } diff --git a/internal/store/persistentvolume_test.go b/internal/store/persistentvolume_test.go index de13969d1c..a5686d1d3c 100644 --- a/internal/store/persistentvolume_test.go +++ b/internal/store/persistentvolume_test.go @@ -18,15 +18,18 @@ package store import ( "testing" + "time" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) func TestPersistentVolumeStore(t *testing.T) { + iscsiInitiatorName := "iqn.my.test.initiator:112233" cases := []generateMetricsTestCase{ // Verify phase enumerations. { @@ -39,7 +42,7 @@ func TestPersistentVolumeStore(t *testing.T) { }, }, Want: ` - # HELP kube_persistentvolume_status_phase The phase indicates if a volume is available, bound to a claim, or released by a claim. + # HELP kube_persistentvolume_status_phase [STABLE] The phase indicates if a volume is available, bound to a claim, or released by a claim. # TYPE kube_persistentvolume_status_phase gauge kube_persistentvolume_status_phase{persistentvolume="test-pv-pending",phase="Available"} 0 kube_persistentvolume_status_phase{persistentvolume="test-pv-pending",phase="Bound"} 0 @@ -61,7 +64,7 @@ func TestPersistentVolumeStore(t *testing.T) { }, }, Want: ` - # HELP kube_persistentvolume_status_phase The phase indicates if a volume is available, bound to a claim, or released by a claim. + # HELP kube_persistentvolume_status_phase [STABLE] The phase indicates if a volume is available, bound to a claim, or released by a claim. # TYPE kube_persistentvolume_status_phase gauge kube_persistentvolume_status_phase{persistentvolume="test-pv-available",phase="Available"} 1 kube_persistentvolume_status_phase{persistentvolume="test-pv-available",phase="Bound"} 0 @@ -81,7 +84,7 @@ func TestPersistentVolumeStore(t *testing.T) { }, }, Want: ` - # HELP kube_persistentvolume_status_phase The phase indicates if a volume is available, bound to a claim, or released by a claim. + # HELP kube_persistentvolume_status_phase [STABLE] The phase indicates if a volume is available, bound to a claim, or released by a claim. # TYPE kube_persistentvolume_status_phase gauge kube_persistentvolume_status_phase{persistentvolume="test-pv-bound",phase="Available"} 0 kube_persistentvolume_status_phase{persistentvolume="test-pv-bound",phase="Bound"} 1 @@ -101,7 +104,7 @@ func TestPersistentVolumeStore(t *testing.T) { }, }, Want: ` - # HELP kube_persistentvolume_status_phase The phase indicates if a volume is available, bound to a claim, or released by a claim. + # HELP kube_persistentvolume_status_phase [STABLE] The phase indicates if a volume is available, bound to a claim, or released by a claim. # TYPE kube_persistentvolume_status_phase gauge kube_persistentvolume_status_phase{persistentvolume="test-pv-released",phase="Available"} 0 kube_persistentvolume_status_phase{persistentvolume="test-pv-released",phase="Bound"} 0 @@ -122,7 +125,7 @@ func TestPersistentVolumeStore(t *testing.T) { }, }, Want: ` - # HELP kube_persistentvolume_status_phase The phase indicates if a volume is available, bound to a claim, or released by a claim. + # HELP kube_persistentvolume_status_phase [STABLE] The phase indicates if a volume is available, bound to a claim, or released by a claim. # TYPE kube_persistentvolume_status_phase gauge kube_persistentvolume_status_phase{persistentvolume="test-pv-failed",phase="Available"} 0 kube_persistentvolume_status_phase{persistentvolume="test-pv-failed",phase="Bound"} 0 @@ -145,7 +148,7 @@ func TestPersistentVolumeStore(t *testing.T) { }, }, Want: ` - # HELP kube_persistentvolume_status_phase The phase indicates if a volume is available, bound to a claim, or released by a claim. + # HELP kube_persistentvolume_status_phase [STABLE] The phase indicates if a volume is available, bound to a claim, or released by a claim. # TYPE kube_persistentvolume_status_phase gauge kube_persistentvolume_status_phase{persistentvolume="test-pv-pending",phase="Available"} 0 kube_persistentvolume_status_phase{persistentvolume="test-pv-pending",phase="Bound"} 0 @@ -167,9 +170,337 @@ func TestPersistentVolumeStore(t *testing.T) { }, }, Want: ` - # HELP kube_persistentvolume_info Information about persistentvolume. + # HELP kube_persistentvolume_info [STABLE] Information about persistentvolume. # TYPE kube_persistentvolume_info gauge - kube_persistentvolume_info{persistentvolume="test-pv-available",storageclass=""} 1 + kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1 + `, + MetricNames: []string{"kube_persistentvolume_info"}, + }, + { + Obj: &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pv-available", + Labels: map[string]string{ + "fc_lun": "456", + }, + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumeAvailable, + }, + }, + Want: ` + # HELP kube_persistentvolume_info [STABLE] Information about persistentvolume. + # TYPE kube_persistentvolume_info gauge + kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1 + `, + MetricNames: []string{"kube_persistentvolume_info"}, + }, + { + Obj: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ + PDName: "name", + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pv-available", + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumeAvailable, + }, + }, + Want: ` + # HELP kube_persistentvolume_info [STABLE] Information about persistentvolume. + # TYPE kube_persistentvolume_info gauge + kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="name",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1 + `, + MetricNames: []string{"kube_persistentvolume_info"}, + }, + { + Obj: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{ + VolumeID: "aws://eu-west-1c/vol-012d34d567890123b", + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pv-available", + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumeAvailable, + }, + }, + Want: ` + # HELP kube_persistentvolume_info [STABLE] Information about persistentvolume. + # TYPE kube_persistentvolume_info gauge + kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="aws://eu-west-1c/vol-012d34d567890123b",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1 + `, + MetricNames: []string{"kube_persistentvolume_info"}, + }, + { + Obj: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + AzureDisk: &v1.AzureDiskVolumeSource{ + DiskName: "azure_disk_1", + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pv-available", + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumeAvailable, + }, + }, + Want: ` + # HELP kube_persistentvolume_info [STABLE] Information about persistentvolume. + # TYPE kube_persistentvolume_info gauge + kube_persistentvolume_info{azure_disk_name="azure_disk_1",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1 + `, + MetricNames: []string{"kube_persistentvolume_info"}, + }, + { + Obj: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + FC: &v1.FCVolumeSource{ + Lun: int32ptr(123), + TargetWWNs: []string{"0123456789abcdef", "abcdef0123456789"}, + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pv-available", + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumeAvailable, + }, + }, + Want: ` + # HELP kube_persistentvolume_info [STABLE] Information about persistentvolume. + # TYPE kube_persistentvolume_info gauge + kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="123",fc_target_wwns="0123456789abcdef,abcdef0123456789",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1 + `, + MetricNames: []string{"kube_persistentvolume_info"}, + }, + { + Obj: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + FC: &v1.FCVolumeSource{ + WWIDs: []string{"0123456789abcdef", "abcdef0123456789"}, + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pv-available", + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumeAvailable, + }, + }, + Want: ` + # HELP kube_persistentvolume_info [STABLE] Information about persistentvolume. + # TYPE kube_persistentvolume_info gauge + kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="0123456789abcdef,abcdef0123456789",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1 + `, + MetricNames: []string{"kube_persistentvolume_info"}, + }, + { + Obj: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + ISCSI: &v1.ISCSIPersistentVolumeSource{ + TargetPortal: "1.2.3.4:3260", + IQN: "iqn.my.test.server.target00", + Lun: int32(123), + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pv-available", + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumeAvailable, + }, + }, + Want: ` + # HELP kube_persistentvolume_info [STABLE] Information about persistentvolume. + # TYPE kube_persistentvolume_info gauge + kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="iqn.my.test.server.target00",iscsi_lun="123",iscsi_target_portal="1.2.3.4:3260",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1 + `, + MetricNames: []string{"kube_persistentvolume_info"}, + }, + { + Obj: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + ISCSI: &v1.ISCSIPersistentVolumeSource{ + TargetPortal: "1.2.3.4:3260", + IQN: "iqn.my.test.server.target00", + Lun: int32(123), + InitiatorName: &iscsiInitiatorName, + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pv-available", + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumeAvailable, + }, + }, + Want: ` + # HELP kube_persistentvolume_info [STABLE] Information about persistentvolume. + # TYPE kube_persistentvolume_info gauge + kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="iqn.my.test.initiator:112233",iscsi_iqn="iqn.my.test.server.target00",iscsi_lun="123",iscsi_target_portal="1.2.3.4:3260",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1 + `, + MetricNames: []string{"kube_persistentvolume_info"}, + }, + { + Obj: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + NFS: &v1.NFSVolumeSource{ + Server: "1.2.3.4", + Path: "/myPath", + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pv-available", + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumeAvailable, + }, + }, + Want: ` + # HELP kube_persistentvolume_info [STABLE] Information about persistentvolume. + # TYPE kube_persistentvolume_info gauge + kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="/myPath",nfs_server="1.2.3.4",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1 + `, + MetricNames: []string{"kube_persistentvolume_info"}, + }, + { + Obj: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + CSI: &v1.CSIPersistentVolumeSource{ + Driver: "test-driver", + VolumeHandle: "test-volume-handle", + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pv-available", + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumeAvailable, + }, + }, + Want: ` + # HELP kube_persistentvolume_info [STABLE] Information about persistentvolume. + # TYPE kube_persistentvolume_info gauge + kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="test-driver",csi_volume_handle="test-volume-handle",persistentvolume="test-pv-available",storageclass=""} 1 + `, + MetricNames: []string{"kube_persistentvolume_info"}, + }, + { + Obj: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + Local: &v1.LocalVolumeSource{ + FSType: pointer.String("ext4"), + Path: "/mnt/data", + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pv-available", + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumeAvailable, + }, + }, + Want: ` + # HELP kube_persistentvolume_info [STABLE] Information about persistentvolume. + # TYPE kube_persistentvolume_info gauge + kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="/mnt/data",local_fs="ext4",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1 + `, + MetricNames: []string{"kube_persistentvolume_info"}, + }, + { + Obj: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + Local: &v1.LocalVolumeSource{ + Path: "/mnt/data", + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pv-available", + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumeAvailable, + }, + }, + Want: ` + # HELP kube_persistentvolume_info [STABLE] Information about persistentvolume. + # TYPE kube_persistentvolume_info gauge + kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="/mnt/data",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1 + `, + MetricNames: []string{"kube_persistentvolume_info"}, + }, + { + Obj: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/mnt/data", + Type: hostPathTypePointer(v1.HostPathDirectory), + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pv-available", + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumeAvailable, + }, + }, + Want: ` + # HELP kube_persistentvolume_info [STABLE] Information about persistentvolume. + # TYPE kube_persistentvolume_info gauge + kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="/mnt/data",host_path_type="Directory",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1 + `, + MetricNames: []string{"kube_persistentvolume_info"}, + }, + { + Obj: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/mnt/data", + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pv-available", + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumeAvailable, + }, + }, + Want: ` + # HELP kube_persistentvolume_info [STABLE] Information about persistentvolume. + # TYPE kube_persistentvolume_info gauge + kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="/mnt/data",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1 `, MetricNames: []string{"kube_persistentvolume_info"}, }, @@ -189,9 +520,9 @@ func TestPersistentVolumeStore(t *testing.T) { }, }, Want: ` - # HELP kube_persistentvolume_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_persistentvolume_labels [STABLE] Kubernetes labels converted to Prometheus labels. # TYPE kube_persistentvolume_labels gauge - kube_persistentvolume_labels{label_app="mysql-server",persistentvolume="test-labeled-pv"} 1 + kube_persistentvolume_labels{persistentvolume="test-labeled-pv"} 1 `, MetricNames: []string{"kube_persistentvolume_labels"}, }, @@ -205,12 +536,52 @@ func TestPersistentVolumeStore(t *testing.T) { }, }, Want: ` - # HELP kube_persistentvolume_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_persistentvolume_labels [STABLE] Kubernetes labels converted to Prometheus labels. # TYPE kube_persistentvolume_labels gauge kube_persistentvolume_labels{persistentvolume="test-unlabeled-pv"} 1 `, MetricNames: []string{"kube_persistentvolume_labels"}, }, + { + Obj: &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-claimed-pv", + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumePending, + }, + Spec: v1.PersistentVolumeSpec{ + StorageClassName: "test", + ClaimRef: &v1.ObjectReference{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + Name: "pv-claim", + Namespace: "default", + }, + }, + }, + Want: ` + # HELP kube_persistentvolume_claim_ref [STABLE] Information about the Persistent Volume Claim Reference. + # TYPE kube_persistentvolume_claim_ref gauge + kube_persistentvolume_claim_ref{claim_namespace="default",name="pv-claim",persistentvolume="test-claimed-pv"} 1 + `, + MetricNames: []string{"kube_persistentvolume_claim_ref"}, + }, + { + Obj: &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-unclaimed-pv", + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumeAvailable, + }, + }, + Want: ` + # HELP kube_persistentvolume_claim_ref [STABLE] Information about the Persistent Volume Claim Reference. + # TYPE kube_persistentvolume_claim_ref gauge + `, + MetricNames: []string{"kube_persistentvolume_claim_ref"}, + }, { Obj: &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ @@ -223,18 +594,105 @@ func TestPersistentVolumeStore(t *testing.T) { }, }, Want: ` - # HELP kube_persistentvolume_capacity_bytes Persistentvolume capacity in bytes. + # HELP kube_persistentvolume_capacity_bytes [STABLE] Persistentvolume capacity in bytes. # TYPE kube_persistentvolume_capacity_bytes gauge kube_persistentvolume_capacity_bytes{persistentvolume="test-pv"} 5.36870912e+09 `, MetricNames: []string{"kube_persistentvolume_capacity_bytes"}, }, + { + AllowAnnotationsList: []string{ + "app.k8s.io/owner", + }, + AllowLabelsList: []string{ + "app", + }, + Obj: &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-allowlisted-labels-annotations", + Annotations: map[string]string{ + "app.k8s.io/owner": "mysql-server", + "foo": "bar", + }, + Labels: map[string]string{ + "app": "mysql-server", + "hello": "world", + }, + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumePending, + }, + }, + Want: ` + # HELP kube_persistentvolume_annotations Kubernetes annotations converted to Prometheus labels. + # HELP kube_persistentvolume_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # TYPE kube_persistentvolume_annotations gauge + # TYPE kube_persistentvolume_labels gauge + kube_persistentvolume_annotations{annotation_app_k8s_io_owner="mysql-server",persistentvolume="test-allowlisted-labels-annotations"} 1 + kube_persistentvolume_labels{label_app="mysql-server",persistentvolume="test-allowlisted-labels-annotations"} 1 +`, + MetricNames: []string{ + "kube_persistentvolume_annotations", + "kube_persistentvolume_labels", + }, + }, + { + Obj: &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-defaul-labels-annotations", + Annotations: map[string]string{ + "app.k8s.io/owner": "mysql-server", + "foo": "bar", + }, + Labels: map[string]string{ + "app": "mysql-server", + "hello": "world", + }, + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumePending, + }, + }, + Want: ` + # HELP kube_persistentvolume_annotations Kubernetes annotations converted to Prometheus labels. + # HELP kube_persistentvolume_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # TYPE kube_persistentvolume_annotations gauge + # TYPE kube_persistentvolume_labels gauge + kube_persistentvolume_annotations{persistentvolume="test-defaul-labels-annotations"} 1 + kube_persistentvolume_labels{persistentvolume="test-defaul-labels-annotations"} 1 +`, + MetricNames: []string{ + "kube_persistentvolume_annotations", + "kube_persistentvolume_labels", + }, + }, + { + Obj: &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pv-created", + CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumePending, + }, + }, + Want: ` + # HELP kube_persistentvolume_created Unix creation timestamp + # TYPE kube_persistentvolume_created gauge + kube_persistentvolume_created{persistentvolume="test-pv-created"} 1.5e+09 +`, + MetricNames: []string{"kube_persistentvolume_created"}, + }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(persistentVolumeMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(persistentVolumeMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(persistentVolumeMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) + c.Headers = generator.ExtractMetricFamilyHeaders(persistentVolumeMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } } } + +func hostPathTypePointer(p v1.HostPathType) *v1.HostPathType { + return &p +} diff --git a/internal/store/persistentvolumeclaim.go b/internal/store/persistentvolumeclaim.go index 69d816b385..637382d260 100644 --- a/internal/store/persistentvolumeclaim.go +++ b/internal/store/persistentvolumeclaim.go @@ -17,7 +17,12 @@ limitations under the License. package store import ( - "k8s.io/kube-state-metrics/pkg/metric" + "context" + + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -28,17 +33,23 @@ import ( ) var ( + descPersistentVolumeClaimAnnotationsName = "kube_persistentvolumeclaim_annotations" + descPersistentVolumeClaimAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." descPersistentVolumeClaimLabelsName = "kube_persistentvolumeclaim_labels" descPersistentVolumeClaimLabelsHelp = "Kubernetes labels converted to Prometheus labels." descPersistentVolumeClaimLabelsDefaultLabels = []string{"namespace", "persistentvolumeclaim"} +) - persistentVolumeClaimMetricFamilies = []metric.FamilyGenerator{ - { - Name: descPersistentVolumeClaimLabelsName, - Type: metric.Gauge, - Help: descPersistentVolumeClaimLabelsHelp, - GenerateFunc: wrapPersistentVolumeClaimFunc(func(p *v1.PersistentVolumeClaim) *metric.Family { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(p.Labels) +func persistentVolumeClaimMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + descPersistentVolumeClaimLabelsName, + descPersistentVolumeClaimLabelsHelp, + metric.Gauge, + basemetrics.STABLE, + "", + wrapPersistentVolumeClaimFunc(func(p *v1.PersistentVolumeClaim) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", p.Labels, allowLabelsList) return &metric.Family{ Metrics: []*metric.Metric{ { @@ -49,12 +60,33 @@ var ( }, } }), - }, - { - Name: "kube_persistentvolumeclaim_info", - Type: metric.Gauge, - Help: "Information about persistent volume claim.", - GenerateFunc: wrapPersistentVolumeClaimFunc(func(p *v1.PersistentVolumeClaim) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + descPersistentVolumeClaimAnnotationsName, + descPersistentVolumeClaimAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPersistentVolumeClaimFunc(func(p *v1.PersistentVolumeClaim) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", p.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_persistentvolumeclaim_info", + "Information about persistent volume claim.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPersistentVolumeClaimFunc(func(p *v1.PersistentVolumeClaim) *metric.Family { storageClassName := getPersistentVolumeClaimClass(p) volumeName := p.Spec.VolumeName return &metric.Family{ @@ -67,12 +99,14 @@ var ( }, } }), - }, - { - Name: "kube_persistentvolumeclaim_status_phase", - Type: metric.Gauge, - Help: "The phase the persistent volume claim is currently in.", - GenerateFunc: wrapPersistentVolumeClaimFunc(func(p *v1.PersistentVolumeClaim) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_persistentvolumeclaim_status_phase", + "The phase the persistent volume claim is currently in.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPersistentVolumeClaimFunc(func(p *v1.PersistentVolumeClaim) *metric.Family { phase := p.Status.Phase if phase == "" { @@ -105,12 +139,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_persistentvolumeclaim_resource_requests_storage_bytes", - Type: metric.Gauge, - Help: "The capacity of storage requested by the persistent volume claim.", - GenerateFunc: wrapPersistentVolumeClaimFunc(func(p *v1.PersistentVolumeClaim) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_persistentvolumeclaim_resource_requests_storage_bytes", + "The capacity of storage requested by the persistent volume claim.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPersistentVolumeClaimFunc(func(p *v1.PersistentVolumeClaim) *metric.Family { ms := []*metric.Metric{} if storage, ok := p.Spec.Resources.Requests[v1.ResourceStorage]; ok { @@ -123,12 +159,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_persistentvolumeclaim_access_mode", - Type: metric.Gauge, - Help: "The access mode(s) specified by the persistent volume claim.", - GenerateFunc: wrapPersistentVolumeClaimFunc(func(p *v1.PersistentVolumeClaim) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_persistentvolumeclaim_access_mode", + "The access mode(s) specified by the persistent volume claim.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPersistentVolumeClaimFunc(func(p *v1.PersistentVolumeClaim) *metric.Family { ms := make([]*metric.Metric, len(p.Spec.AccessModes)) for i, mode := range p.Spec.AccessModes { @@ -143,12 +181,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_persistentvolumeclaim_status_condition", - Help: "Information about status of different conditions of persistent volume claim.", - Type: metric.Gauge, - GenerateFunc: wrapPersistentVolumeClaimFunc(func(p *v1.PersistentVolumeClaim) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_persistentvolumeclaim_status_condition", + "Information about status of different conditions of persistent volume claim.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPersistentVolumeClaimFunc(func(p *v1.PersistentVolumeClaim) *metric.Family { ms := make([]*metric.Metric, len(p.Status.Conditions)*len(conditionStatuses)) for i, c := range p.Status.Conditions { @@ -168,9 +208,31 @@ var ( Metrics: ms, } }), - }, + ), + *generator.NewFamilyGeneratorWithStability( + "kube_persistentvolumeclaim_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPersistentVolumeClaimFunc(func(p *v1.PersistentVolumeClaim) *metric.Family { + ms := []*metric.Metric{} + + if !p.CreationTimestamp.IsZero() { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64(p.CreationTimestamp.Unix()), + }) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ), } -) +} func wrapPersistentVolumeClaimFunc(f func(*v1.PersistentVolumeClaim) *metric.Family) func(interface{}) *metric.Family { return func(obj interface{}) *metric.Family { @@ -179,21 +241,22 @@ func wrapPersistentVolumeClaimFunc(f func(*v1.PersistentVolumeClaim) *metric.Fam metricFamily := f(persistentVolumeClaim) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descPersistentVolumeClaimLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{persistentVolumeClaim.Namespace, persistentVolumeClaim.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descPersistentVolumeClaimLabelsDefaultLabels, []string{persistentVolumeClaim.Namespace, persistentVolumeClaim.Name}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createPersistentVolumeClaimListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createPersistentVolumeClaimListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.CoreV1().PersistentVolumeClaims(ns).List(opts) + opts.FieldSelector = fieldSelector + return kubeClient.CoreV1().PersistentVolumeClaims(ns).List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.CoreV1().PersistentVolumeClaims(ns).Watch(opts) + opts.FieldSelector = fieldSelector + return kubeClient.CoreV1().PersistentVolumeClaims(ns).Watch(context.TODO(), opts) }, } } @@ -210,6 +273,6 @@ func getPersistentVolumeClaimClass(claim *v1.PersistentVolumeClaim) string { return *claim.Spec.StorageClassName } - // Special non-empty string to indicate absence of storage class. - return "" + // An empty string indicates the absence of storage class. + return "" } diff --git a/internal/store/persistentvolumeclaim_test.go b/internal/store/persistentvolumeclaim_test.go index 20aad22db3..39c5c6d824 100644 --- a/internal/store/persistentvolumeclaim_test.go +++ b/internal/store/persistentvolumeclaim_test.go @@ -18,12 +18,13 @@ package store import ( "testing" + "time" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) func TestPersistentVolumeClaimStore(t *testing.T) { @@ -31,10 +32,87 @@ func TestPersistentVolumeClaimStore(t *testing.T) { cases := []generateMetricsTestCase{ // Verify phase enumerations. { + AllowAnnotationsList: []string{ + "app.k8s.io/owner", + }, + Obj: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mysql-data", + CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, + Namespace: "default", + Annotations: map[string]string{ + "app": "mysql-server", + "app.k8s.io/owner": "@foo", + }, + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + StorageClassName: &storageClassName, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + VolumeName: "pvc-mysql-data", + }, + Status: v1.PersistentVolumeClaimStatus{ + Phase: v1.ClaimBound, + Conditions: []v1.PersistentVolumeClaimCondition{ + {Type: v1.PersistentVolumeClaimResizing, Status: v1.ConditionTrue}, + {Type: v1.PersistentVolumeClaimFileSystemResizePending, Status: v1.ConditionFalse}, + {Type: v1.PersistentVolumeClaimConditionType("CustomizedType"), Status: v1.ConditionTrue}, + }, + }, + }, + Want: ` + # HELP kube_persistentvolumeclaim_created Unix creation timestamp + # HELP kube_persistentvolumeclaim_access_mode [STABLE] The access mode(s) specified by the persistent volume claim. + # HELP kube_persistentvolumeclaim_annotations Kubernetes annotations converted to Prometheus labels. + # HELP kube_persistentvolumeclaim_info [STABLE] Information about persistent volume claim. + # HELP kube_persistentvolumeclaim_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # HELP kube_persistentvolumeclaim_resource_requests_storage_bytes [STABLE] The capacity of storage requested by the persistent volume claim. + # HELP kube_persistentvolumeclaim_status_phase [STABLE] The phase the persistent volume claim is currently in. + # HELP kube_persistentvolumeclaim_status_condition Information about status of different conditions of persistent volume claim. + # TYPE kube_persistentvolumeclaim_created gauge + # TYPE kube_persistentvolumeclaim_access_mode gauge + # TYPE kube_persistentvolumeclaim_annotations gauge + # TYPE kube_persistentvolumeclaim_info gauge + # TYPE kube_persistentvolumeclaim_labels gauge + # TYPE kube_persistentvolumeclaim_resource_requests_storage_bytes gauge + # TYPE kube_persistentvolumeclaim_status_phase gauge + # TYPE kube_persistentvolumeclaim_status_condition gauge + kube_persistentvolumeclaim_created{namespace="default",persistentvolumeclaim="mysql-data"} 1.5e+09 + kube_persistentvolumeclaim_info{namespace="default",persistentvolumeclaim="mysql-data",storageclass="rbd",volumename="pvc-mysql-data"} 1 + kube_persistentvolumeclaim_status_phase{namespace="default",persistentvolumeclaim="mysql-data",phase="Bound"} 1 + kube_persistentvolumeclaim_status_phase{namespace="default",persistentvolumeclaim="mysql-data",phase="Lost"} 0 + kube_persistentvolumeclaim_status_phase{namespace="default",persistentvolumeclaim="mysql-data",phase="Pending"} 0 + kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="default",persistentvolumeclaim="mysql-data"} 1.073741824e+09 + kube_persistentvolumeclaim_annotations{annotation_app_k8s_io_owner="@foo",namespace="default",persistentvolumeclaim="mysql-data"} 1 + kube_persistentvolumeclaim_labels{namespace="default",persistentvolumeclaim="mysql-data"} 1 + kube_persistentvolumeclaim_access_mode{namespace="default",persistentvolumeclaim="mysql-data",access_mode="ReadWriteOnce"} 1 + kube_persistentvolumeclaim_status_condition{namespace="default",persistentvolumeclaim="mysql-data",status="false",condition="CustomizedType"} 0 + kube_persistentvolumeclaim_status_condition{namespace="default",persistentvolumeclaim="mysql-data",status="false",condition="FileSystemResizePending"} 1 + kube_persistentvolumeclaim_status_condition{namespace="default",persistentvolumeclaim="mysql-data",status="false",condition="Resizing"} 0 + kube_persistentvolumeclaim_status_condition{namespace="default",persistentvolumeclaim="mysql-data",status="true",condition="CustomizedType"} 1 + kube_persistentvolumeclaim_status_condition{namespace="default",persistentvolumeclaim="mysql-data",status="true",condition="FileSystemResizePending"} 0 + kube_persistentvolumeclaim_status_condition{namespace="default",persistentvolumeclaim="mysql-data",status="true",condition="Resizing"} 1 + kube_persistentvolumeclaim_status_condition{namespace="default",persistentvolumeclaim="mysql-data",status="unknown",condition="CustomizedType"} 0 + kube_persistentvolumeclaim_status_condition{namespace="default",persistentvolumeclaim="mysql-data",status="unknown",condition="FileSystemResizePending"} 0 + kube_persistentvolumeclaim_status_condition{namespace="default",persistentvolumeclaim="mysql-data",status="unknown",condition="Resizing"} 0 +`, + MetricNames: []string{"kube_persistentvolumeclaim_info", "kube_persistentvolumeclaim_status_phase", "kube_persistentvolumeclaim_resource_requests_storage_bytes", "kube_persistentvolumeclaim_annotations", "kube_persistentvolumeclaim_labels", "kube_persistentvolumeclaim_access_mode", "kube_persistentvolumeclaim_status_condition", "kube_persistentvolumeclaim_created"}, + }, + { + AllowLabelsList: []string{ + "app", + }, Obj: &v1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: "mysql-data", - Namespace: "default", + Name: "mysql-data", + CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, + Namespace: "default", Labels: map[string]string{ "app": "mysql-server", }, @@ -61,23 +139,29 @@ func TestPersistentVolumeClaimStore(t *testing.T) { }, }, Want: ` - # HELP kube_persistentvolumeclaim_access_mode The access mode(s) specified by the persistent volume claim. - # HELP kube_persistentvolumeclaim_info Information about persistent volume claim. - # HELP kube_persistentvolumeclaim_labels Kubernetes labels converted to Prometheus labels. - # HELP kube_persistentvolumeclaim_resource_requests_storage_bytes The capacity of storage requested by the persistent volume claim. - # HELP kube_persistentvolumeclaim_status_phase The phase the persistent volume claim is currently in. + # HELP kube_persistentvolumeclaim_created Unix creation timestamp + # HELP kube_persistentvolumeclaim_access_mode [STABLE] The access mode(s) specified by the persistent volume claim. + # HELP kube_persistentvolumeclaim_annotations Kubernetes annotations converted to Prometheus labels. + # HELP kube_persistentvolumeclaim_info [STABLE] Information about persistent volume claim. + # HELP kube_persistentvolumeclaim_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # HELP kube_persistentvolumeclaim_resource_requests_storage_bytes [STABLE] The capacity of storage requested by the persistent volume claim. + # HELP kube_persistentvolumeclaim_status_phase [STABLE] The phase the persistent volume claim is currently in. # HELP kube_persistentvolumeclaim_status_condition Information about status of different conditions of persistent volume claim. + # TYPE kube_persistentvolumeclaim_created gauge # TYPE kube_persistentvolumeclaim_access_mode gauge + # TYPE kube_persistentvolumeclaim_annotations gauge # TYPE kube_persistentvolumeclaim_info gauge # TYPE kube_persistentvolumeclaim_labels gauge # TYPE kube_persistentvolumeclaim_resource_requests_storage_bytes gauge # TYPE kube_persistentvolumeclaim_status_phase gauge # TYPE kube_persistentvolumeclaim_status_condition gauge + kube_persistentvolumeclaim_created{namespace="default",persistentvolumeclaim="mysql-data"} 1.5e+09 kube_persistentvolumeclaim_info{namespace="default",persistentvolumeclaim="mysql-data",storageclass="rbd",volumename="pvc-mysql-data"} 1 kube_persistentvolumeclaim_status_phase{namespace="default",persistentvolumeclaim="mysql-data",phase="Bound"} 1 kube_persistentvolumeclaim_status_phase{namespace="default",persistentvolumeclaim="mysql-data",phase="Lost"} 0 kube_persistentvolumeclaim_status_phase{namespace="default",persistentvolumeclaim="mysql-data",phase="Pending"} 0 kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="default",persistentvolumeclaim="mysql-data"} 1.073741824e+09 + kube_persistentvolumeclaim_annotations{namespace="default",persistentvolumeclaim="mysql-data"} 1 kube_persistentvolumeclaim_labels{label_app="mysql-server",namespace="default",persistentvolumeclaim="mysql-data"} 1 kube_persistentvolumeclaim_access_mode{namespace="default",persistentvolumeclaim="mysql-data",access_mode="ReadWriteOnce"} 1 kube_persistentvolumeclaim_status_condition{namespace="default",persistentvolumeclaim="mysql-data",status="false",condition="CustomizedType"} 0 @@ -90,13 +174,14 @@ func TestPersistentVolumeClaimStore(t *testing.T) { kube_persistentvolumeclaim_status_condition{namespace="default",persistentvolumeclaim="mysql-data",status="unknown",condition="FileSystemResizePending"} 0 kube_persistentvolumeclaim_status_condition{namespace="default",persistentvolumeclaim="mysql-data",status="unknown",condition="Resizing"} 0 `, - MetricNames: []string{"kube_persistentvolumeclaim_info", "kube_persistentvolumeclaim_status_phase", "kube_persistentvolumeclaim_resource_requests_storage_bytes", "kube_persistentvolumeclaim_labels", "kube_persistentvolumeclaim_access_mode", "kube_persistentvolumeclaim_status_condition"}, + MetricNames: []string{"kube_persistentvolumeclaim_info", "kube_persistentvolumeclaim_status_phase", "kube_persistentvolumeclaim_resource_requests_storage_bytes", "kube_persistentvolumeclaim_annotations", "kube_persistentvolumeclaim_labels", "kube_persistentvolumeclaim_access_mode", "kube_persistentvolumeclaim_status_condition", "kube_persistentvolumeclaim_created"}, }, { Obj: &v1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: "prometheus-data", - Namespace: "default", + Name: "prometheus-data", + CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, + Namespace: "default", }, Spec: v1.PersistentVolumeClaimSpec{ AccessModes: []v1.PersistentVolumeAccessMode{ @@ -110,18 +195,21 @@ func TestPersistentVolumeClaimStore(t *testing.T) { }, }, Want: ` - # HELP kube_persistentvolumeclaim_access_mode The access mode(s) specified by the persistent volume claim. - # HELP kube_persistentvolumeclaim_info Information about persistent volume claim. - # HELP kube_persistentvolumeclaim_labels Kubernetes labels converted to Prometheus labels. - # HELP kube_persistentvolumeclaim_resource_requests_storage_bytes The capacity of storage requested by the persistent volume claim. - # HELP kube_persistentvolumeclaim_status_phase The phase the persistent volume claim is currently in. + # HELP kube_persistentvolumeclaim_created Unix creation timestamp + # HELP kube_persistentvolumeclaim_access_mode [STABLE] The access mode(s) specified by the persistent volume claim. + # HELP kube_persistentvolumeclaim_info [STABLE] Information about persistent volume claim. + # HELP kube_persistentvolumeclaim_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # HELP kube_persistentvolumeclaim_resource_requests_storage_bytes [STABLE] The capacity of storage requested by the persistent volume claim. + # HELP kube_persistentvolumeclaim_status_phase [STABLE] The phase the persistent volume claim is currently in. # HELP kube_persistentvolumeclaim_status_condition Information about status of different conditions of persistent volume claim. + # TYPE kube_persistentvolumeclaim_created gauge # TYPE kube_persistentvolumeclaim_access_mode gauge # TYPE kube_persistentvolumeclaim_info gauge # TYPE kube_persistentvolumeclaim_labels gauge # TYPE kube_persistentvolumeclaim_resource_requests_storage_bytes gauge # TYPE kube_persistentvolumeclaim_status_phase gauge # TYPE kube_persistentvolumeclaim_status_condition gauge + kube_persistentvolumeclaim_created{namespace="default",persistentvolumeclaim="prometheus-data"} 1.5e+09 kube_persistentvolumeclaim_info{namespace="default",persistentvolumeclaim="prometheus-data",storageclass="rbd",volumename="pvc-prometheus-data"} 1 kube_persistentvolumeclaim_status_phase{namespace="default",persistentvolumeclaim="prometheus-data",phase="Bound"} 0 kube_persistentvolumeclaim_status_phase{namespace="default",persistentvolumeclaim="prometheus-data",phase="Lost"} 0 @@ -129,12 +217,13 @@ func TestPersistentVolumeClaimStore(t *testing.T) { kube_persistentvolumeclaim_labels{namespace="default",persistentvolumeclaim="prometheus-data"} 1 kube_persistentvolumeclaim_access_mode{namespace="default",persistentvolumeclaim="prometheus-data",access_mode="ReadWriteOnce"} 1 `, - MetricNames: []string{"kube_persistentvolumeclaim_info", "kube_persistentvolumeclaim_status_phase", "kube_persistentvolumeclaim_resource_requests_storage_bytes", "kube_persistentvolumeclaim_labels", "kube_persistentvolumeclaim_access_mode", "kube_persistentvolumeclaim_status_condition"}, + MetricNames: []string{"kube_persistentvolumeclaim_info", "kube_persistentvolumeclaim_status_phase", "kube_persistentvolumeclaim_resource_requests_storage_bytes", "kube_persistentvolumeclaim_labels", "kube_persistentvolumeclaim_access_mode", "kube_persistentvolumeclaim_status_condition", "kube_persistentvolumeclaim_created"}, }, { Obj: &v1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: "mongo-data", + Name: "mongo-data", + CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, }, Spec: v1.PersistentVolumeClaimSpec{ AccessModes: []v1.PersistentVolumeAccessMode{ @@ -151,23 +240,29 @@ func TestPersistentVolumeClaimStore(t *testing.T) { }, }, Want: ` - # HELP kube_persistentvolumeclaim_access_mode The access mode(s) specified by the persistent volume claim. - # HELP kube_persistentvolumeclaim_info Information about persistent volume claim. - # HELP kube_persistentvolumeclaim_labels Kubernetes labels converted to Prometheus labels. - # HELP kube_persistentvolumeclaim_resource_requests_storage_bytes The capacity of storage requested by the persistent volume claim. - # HELP kube_persistentvolumeclaim_status_phase The phase the persistent volume claim is currently in. + # HELP kube_persistentvolumeclaim_created Unix creation timestamp + # HELP kube_persistentvolumeclaim_access_mode [STABLE] The access mode(s) specified by the persistent volume claim. + # HELP kube_persistentvolumeclaim_annotations Kubernetes annotations converted to Prometheus labels. + # HELP kube_persistentvolumeclaim_info [STABLE] Information about persistent volume claim. + # HELP kube_persistentvolumeclaim_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # HELP kube_persistentvolumeclaim_resource_requests_storage_bytes [STABLE] The capacity of storage requested by the persistent volume claim. + # HELP kube_persistentvolumeclaim_status_phase [STABLE] The phase the persistent volume claim is currently in. # HELP kube_persistentvolumeclaim_status_condition Information about status of different conditions of persistent volume claim. + # TYPE kube_persistentvolumeclaim_created gauge # TYPE kube_persistentvolumeclaim_access_mode gauge + # TYPE kube_persistentvolumeclaim_annotations gauge # TYPE kube_persistentvolumeclaim_info gauge # TYPE kube_persistentvolumeclaim_labels gauge # TYPE kube_persistentvolumeclaim_resource_requests_storage_bytes gauge # TYPE kube_persistentvolumeclaim_status_phase gauge # TYPE kube_persistentvolumeclaim_status_condition gauge - kube_persistentvolumeclaim_info{namespace="",persistentvolumeclaim="mongo-data",storageclass="",volumename=""} 1 + kube_persistentvolumeclaim_created{namespace="",persistentvolumeclaim="mongo-data"} 1.5e+09 + kube_persistentvolumeclaim_info{namespace="",persistentvolumeclaim="mongo-data",storageclass="",volumename=""} 1 kube_persistentvolumeclaim_status_phase{namespace="",persistentvolumeclaim="mongo-data",phase="Bound"} 0 kube_persistentvolumeclaim_status_phase{namespace="",persistentvolumeclaim="mongo-data",phase="Lost"} 1 kube_persistentvolumeclaim_status_phase{namespace="",persistentvolumeclaim="mongo-data",phase="Pending"} 0 kube_persistentvolumeclaim_labels{namespace="",persistentvolumeclaim="mongo-data"} 1 + kube_persistentvolumeclaim_annotations{namespace="",persistentvolumeclaim="mongo-data"} 1 kube_persistentvolumeclaim_access_mode{namespace="",persistentvolumeclaim="mongo-data",access_mode="ReadWriteOnce"} 1 kube_persistentvolumeclaim_status_condition{namespace="",persistentvolumeclaim="mongo-data",status="false",condition="CustomizedType"} 1 kube_persistentvolumeclaim_status_condition{namespace="",persistentvolumeclaim="mongo-data",status="false",condition="FileSystemResizePending"} 0 @@ -179,12 +274,12 @@ func TestPersistentVolumeClaimStore(t *testing.T) { kube_persistentvolumeclaim_status_condition{namespace="",persistentvolumeclaim="mongo-data",status="unknown",condition="FileSystemResizePending"} 0 kube_persistentvolumeclaim_status_condition{namespace="",persistentvolumeclaim="mongo-data",status="unknown",condition="Resizing"} 0 `, - MetricNames: []string{"kube_persistentvolumeclaim_info", "kube_persistentvolumeclaim_status_phase", "kube_persistentvolumeclaim_resource_requests_storage_bytes", "kube_persistentvolumeclaim_labels", "kube_persistentvolumeclaim_access_mode", "kube_persistentvolumeclaim_status_condition"}, + MetricNames: []string{"kube_persistentvolumeclaim_created", "kube_persistentvolumeclaim_info", "kube_persistentvolumeclaim_status_phase", "kube_persistentvolumeclaim_resource_requests_storage_bytes", "kube_persistentvolumeclaim_annotations", "kube_persistentvolumeclaim_labels", "kube_persistentvolumeclaim_access_mode", "kube_persistentvolumeclaim_status_condition"}, }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(persistentVolumeClaimMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(persistentVolumeClaimMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(persistentVolumeClaimMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) + c.Headers = generator.ExtractMetricFamilyHeaders(persistentVolumeClaimMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/pod.go b/internal/store/pod.go index 49eadef617..f6275d2df4 100644 --- a/internal/store/pod.go +++ b/internal/store/pod.go @@ -17,10 +17,15 @@ limitations under the License. package store import ( + "context" "strconv" - "k8s.io/kube-state-metrics/pkg/constant" - "k8s.io/kube-state-metrics/pkg/metric" + basemetrics "k8s.io/component-base/metrics" + "k8s.io/utils/net" + + "k8s.io/kube-state-metrics/v2/pkg/constant" + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -30,1016 +35,1622 @@ import ( "k8s.io/client-go/tools/cache" ) -const nodeUnreachablePodReason = "NodeLost" - var ( - descPodLabelsDefaultLabels = []string{"namespace", "pod"} - containerWaitingReasons = []string{"ContainerCreating", "CrashLoopBackOff", "CreateContainerConfigError", "ErrImagePull", "ImagePullBackOff", "CreateContainerError", "InvalidImageName"} - containerTerminatedReasons = []string{"OOMKilled", "Completed", "Error", "ContainerCannotRun", "DeadlineExceeded", "Evicted"} - - podMetricFamilies = []metric.FamilyGenerator{ - { - Name: "kube_pod_info", - Type: metric.Gauge, - Help: "Information about pod.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - createdBy := metav1.GetControllerOf(p) - createdByKind := "" - createdByName := "" - if createdBy != nil { - if createdBy.Kind != "" { - createdByKind = createdBy.Kind - } - if createdBy.Name != "" { - createdByName = createdBy.Name + descPodLabelsDefaultLabels = []string{"namespace", "pod", "uid"} + podStatusReasons = []string{"Evicted", "NodeAffinity", "NodeLost", "Shutdown", "UnexpectedAdmissionError"} +) + +func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + createPodCompletionTimeFamilyGenerator(), + createPodContainerInfoFamilyGenerator(), + createPodContainerResourceLimitsFamilyGenerator(), + createPodContainerResourceRequestsFamilyGenerator(), + createPodContainerStateStartedFamilyGenerator(), + createPodContainerStatusLastTerminatedReasonFamilyGenerator(), + createPodContainerStatusLastTerminatedExitCodeFamilyGenerator(), + createPodContainerStatusReadyFamilyGenerator(), + createPodContainerStatusRestartsTotalFamilyGenerator(), + createPodContainerStatusRunningFamilyGenerator(), + createPodContainerStatusTerminatedFamilyGenerator(), + createPodContainerStatusTerminatedReasonFamilyGenerator(), + createPodContainerStatusWaitingFamilyGenerator(), + createPodContainerStatusWaitingReasonFamilyGenerator(), + createPodCreatedFamilyGenerator(), + createPodDeletionTimestampFamilyGenerator(), + createPodInfoFamilyGenerator(), + createPodIPFamilyGenerator(), + createPodInitContainerInfoFamilyGenerator(), + createPodInitContainerResourceLimitsFamilyGenerator(), + createPodInitContainerResourceRequestsFamilyGenerator(), + createPodInitContainerStatusLastTerminatedReasonFamilyGenerator(), + createPodInitContainerStatusReadyFamilyGenerator(), + createPodInitContainerStatusRestartsTotalFamilyGenerator(), + createPodInitContainerStatusRunningFamilyGenerator(), + createPodInitContainerStatusTerminatedFamilyGenerator(), + createPodInitContainerStatusTerminatedReasonFamilyGenerator(), + createPodInitContainerStatusWaitingFamilyGenerator(), + createPodInitContainerStatusWaitingReasonFamilyGenerator(), + createPodAnnotationsGenerator(allowAnnotationsList), + createPodLabelsGenerator(allowLabelsList), + createPodOverheadCPUCoresFamilyGenerator(), + createPodOverheadMemoryBytesFamilyGenerator(), + createPodOwnerFamilyGenerator(), + createPodRestartPolicyFamilyGenerator(), + createPodRuntimeClassNameInfoFamilyGenerator(), + createPodSpecVolumesPersistentVolumeClaimsInfoFamilyGenerator(), + createPodSpecVolumesPersistentVolumeClaimsReadonlyFamilyGenerator(), + createPodStartTimeFamilyGenerator(), + createPodStatusPhaseFamilyGenerator(), + createPodStatusQosClassFamilyGenerator(), + createPodStatusReadyFamilyGenerator(), + createPodStatusReadyTimeFamilyGenerator(), + createPodStatusContainerReadyTimeFamilyGenerator(), + createPodStatusReasonFamilyGenerator(), + createPodStatusScheduledFamilyGenerator(), + createPodStatusScheduledTimeFamilyGenerator(), + createPodStatusUnschedulableFamilyGenerator(), + createPodTolerationsFamilyGenerator(), + createPodNodeSelectorsFamilyGenerator(), + } +} + +func createPodCompletionTimeFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_completion_time", + "Completion time in unix timestamp for a pod.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + var lastFinishTime float64 + for _, cs := range p.Status.ContainerStatuses { + if cs.State.Terminated != nil { + if lastFinishTime == 0 || lastFinishTime < float64(cs.State.Terminated.FinishedAt.Unix()) { + lastFinishTime = float64(cs.State.Terminated.FinishedAt.Unix()) } } + } - m := metric.Metric{ + if lastFinishTime > 0 { + ms = append(ms, &metric.Metric{ - LabelKeys: []string{"host_ip", "pod_ip", "uid", "node", "created_by_kind", "created_by_name", "priority_class"}, - LabelValues: []string{p.Status.HostIP, p.Status.PodIP, string(p.UID), p.Spec.NodeName, createdByKind, createdByName, p.Spec.PriorityClassName}, - Value: 1, - } + LabelKeys: []string{}, + LabelValues: []string{}, + Value: lastFinishTime, + }) + } - return &metric.Family{ - Metrics: []*metric.Metric{&m}, - } - }), - }, - { - Name: "kube_pod_start_time", - Type: metric.Gauge, - Help: "Start time in unix timestamp for a pod.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := []*metric.Metric{} - - if p.Status.StartTime != nil { + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodContainerInfoFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_container_info", + "Information about a container in a pod.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + labelKeys := []string{"container", "image_spec", "image", "image_id", "container_id"} + + for _, c := range p.Spec.Containers { + for _, cs := range p.Status.ContainerStatuses { + if cs.Name != c.Name { + continue + } ms = append(ms, &metric.Metric{ - LabelKeys: []string{}, - LabelValues: []string{}, - Value: float64((p.Status.StartTime).Unix()), + LabelKeys: labelKeys, + LabelValues: []string{cs.Name, c.Image, cs.Image, cs.ImageID, cs.ContainerID}, + Value: 1, }) } + } + return &metric.Family{ + Metrics: ms, + } + }), + ) +} - return &metric.Family{ - Metrics: ms, +func createPodContainerResourceLimitsFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_container_resource_limits", + "The number of requested limit resource by a container. It is recommended to use the kube_pod_resource_limits metric exposed by kube-scheduler instead, as it is more precise.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + for _, c := range p.Spec.Containers { + lim := c.Resources.Limits + + for resourceName, val := range lim { + switch resourceName { + case v1.ResourceCPU: + ms = append(ms, &metric.Metric{ + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitCore)}, + Value: float64(val.MilliValue()) / 1000, + }) + case v1.ResourceStorage: + fallthrough + case v1.ResourceEphemeralStorage: + fallthrough + case v1.ResourceMemory: + ms = append(ms, &metric.Metric{ + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, + Value: float64(val.Value()), + }) + default: + if isHugePageResourceName(resourceName) { + ms = append(ms, &metric.Metric{ + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, + Value: float64(val.Value()), + }) + } + if isAttachableVolumeResourceName(resourceName) { + ms = append(ms, &metric.Metric{ + Value: float64(val.Value()), + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, + }) + } + if isExtendedResourceName(resourceName) { + ms = append(ms, &metric.Metric{ + Value: float64(val.Value()), + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitInteger)}, + }) + + } + } } - }), - }, - { - Name: "kube_pod_completion_time", - Type: metric.Gauge, - Help: "Completion time in unix timestamp for a pod.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := []*metric.Metric{} - - var lastFinishTime float64 - for _, cs := range p.Status.ContainerStatuses { - if cs.State.Terminated != nil { - if lastFinishTime == 0 || lastFinishTime < float64(cs.State.Terminated.FinishedAt.Unix()) { - lastFinishTime = float64(cs.State.Terminated.FinishedAt.Unix()) + } + + for _, metric := range ms { + metric.LabelKeys = []string{"container", "node", "resource", "unit"} + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodContainerResourceRequestsFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_container_resource_requests", + "The number of requested request resource by a container. It is recommended to use the kube_pod_resource_requests metric exposed by kube-scheduler instead, as it is more precise.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + for _, c := range p.Spec.Containers { + req := c.Resources.Requests + + for resourceName, val := range req { + switch resourceName { + case v1.ResourceCPU: + ms = append(ms, &metric.Metric{ + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitCore)}, + Value: float64(val.MilliValue()) / 1000, + }) + case v1.ResourceStorage: + fallthrough + case v1.ResourceEphemeralStorage: + fallthrough + case v1.ResourceMemory: + ms = append(ms, &metric.Metric{ + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, + Value: float64(val.Value()), + }) + default: + if isHugePageResourceName(resourceName) { + ms = append(ms, &metric.Metric{ + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, + Value: float64(val.Value()), + }) + } + if isAttachableVolumeResourceName(resourceName) { + ms = append(ms, &metric.Metric{ + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, + Value: float64(val.Value()), + }) + } + if isExtendedResourceName(resourceName) { + ms = append(ms, &metric.Metric{ + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitInteger)}, + Value: float64(val.Value()), + }) } } } + } - if lastFinishTime > 0 { - ms = append(ms, &metric.Metric{ + for _, metric := range ms { + metric.LabelKeys = []string{"container", "node", "resource", "unit"} + } - LabelKeys: []string{}, - LabelValues: []string{}, - Value: lastFinishTime, + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodContainerStateStartedFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_container_state_started", + "Start time in unix timestamp for a pod container.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + for _, cs := range p.Status.ContainerStatuses { + if cs.State.Running != nil { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"container"}, + LabelValues: []string{cs.Name}, + Value: float64((cs.State.Running.StartedAt).Unix()), + }) + } else if cs.State.Terminated != nil { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"container"}, + LabelValues: []string{cs.Name}, + Value: float64((cs.State.Terminated.StartedAt).Unix()), }) } + } - return &metric.Family{ - Metrics: ms, + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodContainerStatusLastTerminatedReasonFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_container_status_last_terminated_reason", + "Describes the last reason the container was in terminated state.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := make([]*metric.Metric, 0, len(p.Status.ContainerStatuses)) + for _, cs := range p.Status.ContainerStatuses { + if cs.LastTerminationState.Terminated != nil { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"container", "reason"}, + LabelValues: []string{cs.Name, cs.LastTerminationState.Terminated.Reason}, + Value: 1, + }) } - }), - }, - { - Name: "kube_pod_owner", - Type: metric.Gauge, - Help: "Information about the Pod's owner.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - labelKeys := []string{"owner_kind", "owner_name", "owner_is_controller"} - - owners := p.GetOwnerReferences() - if len(owners) == 0 { - return &metric.Family{ - Metrics: []*metric.Metric{ - { - LabelKeys: labelKeys, - LabelValues: []string{"", "", ""}, - Value: 1, - }, - }, - } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodContainerStatusLastTerminatedExitCodeFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_container_status_last_terminated_exitcode", + "Describes the exit code for the last container in terminated state.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := make([]*metric.Metric, 0, len(p.Status.ContainerStatuses)) + for _, cs := range p.Status.ContainerStatuses { + if cs.LastTerminationState.Terminated != nil { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"container"}, + LabelValues: []string{cs.Name}, + Value: float64(cs.LastTerminationState.Terminated.ExitCode), + }) } + } - ms := make([]*metric.Metric, len(owners)) + return &metric.Family{ + Metrics: ms, + } + }), + ) +} - for i, owner := range owners { - if owner.Controller != nil { - ms[i] = &metric.Metric{ - LabelKeys: labelKeys, - LabelValues: []string{owner.Kind, owner.Name, strconv.FormatBool(*owner.Controller)}, - Value: 1, - } - } else { - ms[i] = &metric.Metric{ - LabelKeys: labelKeys, - LabelValues: []string{owner.Kind, owner.Name, "false"}, - Value: 1, - } - } +func createPodContainerStatusReadyFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_container_status_ready", + "Describes whether the containers readiness check succeeded.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := make([]*metric.Metric, len(p.Status.ContainerStatuses)) + for i, cs := range p.Status.ContainerStatuses { + ms[i] = &metric.Metric{ + LabelKeys: []string{"container"}, + LabelValues: []string{cs.Name}, + Value: boolFloat64(cs.Ready), + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodContainerStatusRestartsTotalFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_container_status_restarts_total", + "The number of container restarts per container.", + metric.Counter, basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := make([]*metric.Metric, len(p.Status.ContainerStatuses)) + + for i, cs := range p.Status.ContainerStatuses { + ms[i] = &metric.Metric{ + LabelKeys: []string{"container"}, + LabelValues: []string{cs.Name}, + Value: float64(cs.RestartCount), + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodContainerStatusRunningFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_container_status_running", + "Describes whether the container is currently in running state.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := make([]*metric.Metric, len(p.Status.ContainerStatuses)) + + for i, cs := range p.Status.ContainerStatuses { + ms[i] = &metric.Metric{ + LabelKeys: []string{"container"}, + LabelValues: []string{cs.Name}, + Value: boolFloat64(cs.State.Running != nil), + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodContainerStatusTerminatedFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_container_status_terminated", + "Describes whether the container is currently in terminated state.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := make([]*metric.Metric, len(p.Status.ContainerStatuses)) + + for i, cs := range p.Status.ContainerStatuses { + ms[i] = &metric.Metric{ + LabelKeys: []string{"container"}, + LabelValues: []string{cs.Name}, + Value: boolFloat64(cs.State.Terminated != nil), + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodContainerStatusTerminatedReasonFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_container_status_terminated_reason", + "Describes the reason the container is currently in terminated state.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := make([]*metric.Metric, 0, len(p.Status.ContainerStatuses)) + for _, cs := range p.Status.ContainerStatuses { + if cs.State.Terminated != nil { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"container", "reason"}, + LabelValues: []string{cs.Name, cs.State.Terminated.Reason}, + Value: 1, + }) } + } - return &metric.Family{ - Metrics: ms, + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodContainerStatusWaitingFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_container_status_waiting", + "Describes whether the container is currently in waiting state.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := make([]*metric.Metric, len(p.Status.ContainerStatuses)) + + for i, cs := range p.Status.ContainerStatuses { + ms[i] = &metric.Metric{ + LabelKeys: []string{"container"}, + LabelValues: []string{cs.Name}, + Value: boolFloat64(cs.State.Waiting != nil), + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodContainerStatusWaitingReasonFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_container_status_waiting_reason", + "Describes the reason the container is currently in waiting state.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := make([]*metric.Metric, 0, len(p.Status.ContainerStatuses)) + for _, cs := range p.Status.ContainerStatuses { + // Skip creating series for running containers. + if cs.State.Waiting != nil { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"container", "reason"}, + LabelValues: []string{cs.Name, cs.State.Waiting.Reason}, + Value: 1, + }) } - }), - }, - { - Name: "kube_pod_labels", - Type: metric.Gauge, - Help: "Kubernetes labels converted to Prometheus labels.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(p.Labels) - m := metric.Metric{ + } + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodCreatedFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + if !p.CreationTimestamp.IsZero() { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64(p.CreationTimestamp.Unix()), + }) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodDeletionTimestampFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_deletion_timestamp", + "Unix deletion timestamp", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + if p.DeletionTimestamp != nil && !p.DeletionTimestamp.IsZero() { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64(p.DeletionTimestamp.Unix()), + }) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodInfoFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_info", + "Information about pod.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + createdBy := metav1.GetControllerOf(p) + createdByKind := "" + createdByName := "" + if createdBy != nil { + if createdBy.Kind != "" { + createdByKind = createdBy.Kind + } + if createdBy.Name != "" { + createdByName = createdBy.Name + } + } + + m := metric.Metric{ + LabelKeys: []string{"host_ip", "pod_ip", "node", "created_by_kind", "created_by_name", "priority_class", "host_network"}, + LabelValues: []string{p.Status.HostIP, p.Status.PodIP, p.Spec.NodeName, createdByKind, createdByName, p.Spec.PriorityClassName, strconv.FormatBool(p.Spec.HostNetwork)}, + Value: 1, + } + + return &metric.Family{ + Metrics: []*metric.Metric{&m}, + } + }), + ) +} + +func createPodIPFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_ips", + "Pod IP addresses", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := make([]*metric.Metric, len(p.Status.PodIPs)) + labelKeys := []string{"ip", "ip_family"} + + for i, ip := range p.Status.PodIPs { + netIP := net.ParseIPSloppy(ip.IP) + var ipFamily net.IPFamily + switch { + case net.IsIPv4(netIP): + ipFamily = net.IPv4 + case net.IsIPv6(netIP): + ipFamily = net.IPv6 + default: + continue // nil from ParseIPSloppy indicates failure to parse, so we don't include that in our metrics series + } + ms[i] = &metric.Metric{ LabelKeys: labelKeys, - LabelValues: labelValues, + LabelValues: []string{ip.IP, string(ipFamily)}, Value: 1, } - return &metric.Family{ - Metrics: []*metric.Metric{&m}, - } - }), - }, - { - Name: "kube_pod_created", - Type: metric.Gauge, - Help: "Unix creation timestamp", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := []*metric.Metric{} - - if !p.CreationTimestamp.IsZero() { + } + + return &metric.Family{ + Metrics: ms, + } + })) +} + +func createPodInitContainerInfoFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_init_container_info", + "Information about an init container in a pod.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + labelKeys := []string{"container", "image_spec", "image", "image_id", "container_id"} + + for _, c := range p.Spec.InitContainers { + for _, cs := range p.Status.InitContainerStatuses { + if cs.Name != c.Name { + continue + } ms = append(ms, &metric.Metric{ - LabelKeys: []string{}, - LabelValues: []string{}, - Value: float64(p.CreationTimestamp.Unix()), + LabelKeys: labelKeys, + LabelValues: []string{cs.Name, c.Image, cs.Image, cs.ImageID, cs.ContainerID}, + Value: 1, }) } + } - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_restart_policy", - Type: metric.Gauge, - Help: "Describes the restart policy in use by this pod.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - return &metric.Family{ - Metrics: []*metric.Metric{ - { - LabelKeys: []string{"type"}, - LabelValues: []string{string(p.Spec.RestartPolicy)}, - Value: float64(1), - }, - }, - } - }), - }, - { - Name: "kube_pod_status_scheduled_time", - Type: metric.Gauge, - Help: "Unix timestamp when pod moved into scheduled status", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := []*metric.Metric{} - - for _, c := range p.Status.Conditions { - switch c.Type { - case v1.PodScheduled: - if c.Status == v1.ConditionTrue { + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodInitContainerResourceLimitsFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_init_container_resource_limits", + "The number of requested limit resource by an init container.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + for _, c := range p.Spec.InitContainers { + lim := c.Resources.Limits + + for resourceName, val := range lim { + switch resourceName { + case v1.ResourceCPU: + ms = append(ms, &metric.Metric{ + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitCore)}, + Value: float64(val.MilliValue()) / 1000, + }) + case v1.ResourceStorage: + fallthrough + case v1.ResourceEphemeralStorage: + fallthrough + case v1.ResourceMemory: + ms = append(ms, &metric.Metric{ + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, + Value: float64(val.Value()), + }) + default: + if isHugePageResourceName(resourceName) { ms = append(ms, &metric.Metric{ - LabelKeys: []string{}, - LabelValues: []string{}, - Value: float64(c.LastTransitionTime.Unix()), + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, + Value: float64(val.Value()), + }) + } + if isAttachableVolumeResourceName(resourceName) { + ms = append(ms, &metric.Metric{ + Value: float64(val.Value()), + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, + }) + } + if isExtendedResourceName(resourceName) { + ms = append(ms, &metric.Metric{ + Value: float64(val.Value()), + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitInteger)}, }) + } } } + } - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_status_unschedulable", - Type: metric.Gauge, - Help: "Describes the unschedulable status for the pod.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := []*metric.Metric{} - - for _, c := range p.Status.Conditions { - switch c.Type { - case v1.PodScheduled: - if c.Status == v1.ConditionFalse { + for _, metric := range ms { + metric.LabelKeys = []string{"container", "node", "resource", "unit"} + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodInitContainerResourceRequestsFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_init_container_resource_requests", + "The number of requested request resource by an init container.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + for _, c := range p.Spec.InitContainers { + req := c.Resources.Requests + + for resourceName, val := range req { + switch resourceName { + case v1.ResourceCPU: + ms = append(ms, &metric.Metric{ + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitCore)}, + Value: float64(val.MilliValue()) / 1000, + }) + case v1.ResourceStorage: + fallthrough + case v1.ResourceEphemeralStorage: + fallthrough + case v1.ResourceMemory: + ms = append(ms, &metric.Metric{ + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, + Value: float64(val.Value()), + }) + default: + if isHugePageResourceName(resourceName) { + ms = append(ms, &metric.Metric{ + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, + Value: float64(val.Value()), + }) + } + if isAttachableVolumeResourceName(resourceName) { + ms = append(ms, &metric.Metric{ + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, + Value: float64(val.Value()), + }) + } + if isExtendedResourceName(resourceName) { ms = append(ms, &metric.Metric{ - LabelKeys: []string{}, - LabelValues: []string{}, - Value: 1, + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitInteger)}, + Value: float64(val.Value()), }) } } } + } - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_status_phase", - Type: metric.Gauge, - Help: "The pods current phase.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - phase := p.Status.Phase - if phase == "" { - return &metric.Family{ - Metrics: []*metric.Metric{}, - } - } + for _, metric := range ms { + metric.LabelKeys = []string{"container", "node", "resource", "unit"} + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} - phases := []struct { - v bool - n string - }{ - {phase == v1.PodPending, string(v1.PodPending)}, - {phase == v1.PodSucceeded, string(v1.PodSucceeded)}, - {phase == v1.PodFailed, string(v1.PodFailed)}, - // This logic is directly copied from: https://github.com/kubernetes/kubernetes/blob/d39bfa0d138368bbe72b0eaf434501dcb4ec9908/pkg/printers/internalversion/printers.go#L597-L601 - // For more info, please go to: https://github.com/kubernetes/kube-state-metrics/issues/410 - {phase == v1.PodRunning && !(p.DeletionTimestamp != nil && p.Status.Reason == nodeUnreachablePodReason), string(v1.PodRunning)}, - {phase == v1.PodUnknown || (p.DeletionTimestamp != nil && p.Status.Reason == nodeUnreachablePodReason), string(v1.PodUnknown)}, +func createPodInitContainerStatusLastTerminatedReasonFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_init_container_status_last_terminated_reason", + "Describes the last reason the init container was in terminated state.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := make([]*metric.Metric, 0, len(p.Status.InitContainerStatuses)) + for _, cs := range p.Status.InitContainerStatuses { + if cs.LastTerminationState.Terminated != nil { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"container", "reason"}, + LabelValues: []string{cs.Name, cs.LastTerminationState.Terminated.Reason}, + Value: 1, + }) } + } + return &metric.Family{ + Metrics: ms, + } + }), + ) +} - ms := make([]*metric.Metric, len(phases)) +func createPodInitContainerStatusReadyFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_init_container_status_ready", + "Describes whether the init containers readiness check succeeded.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses)) + + for i, cs := range p.Status.InitContainerStatuses { + ms[i] = &metric.Metric{ + LabelKeys: []string{"container"}, + LabelValues: []string{cs.Name}, + Value: boolFloat64(cs.Ready), + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} - for i, p := range phases { - ms[i] = &metric.Metric{ +func createPodInitContainerStatusRestartsTotalFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_init_container_status_restarts_total", + "The number of restarts for the init container.", + metric.Counter, basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses)) + + for i, cs := range p.Status.InitContainerStatuses { + ms[i] = &metric.Metric{ + LabelKeys: []string{"container"}, + LabelValues: []string{cs.Name}, + Value: float64(cs.RestartCount), + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} - LabelKeys: []string{"phase"}, - LabelValues: []string{p.n}, - Value: boolFloat64(p.v), - } +func createPodInitContainerStatusRunningFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_init_container_status_running", + "Describes whether the init container is currently in running state.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses)) + + for i, cs := range p.Status.InitContainerStatuses { + ms[i] = &metric.Metric{ + LabelKeys: []string{"container"}, + LabelValues: []string{cs.Name}, + Value: boolFloat64(cs.State.Running != nil), + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodInitContainerStatusTerminatedFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_init_container_status_terminated", + "Describes whether the init container is currently in terminated state.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses)) + + for i, cs := range p.Status.InitContainerStatuses { + ms[i] = &metric.Metric{ + LabelKeys: []string{"container"}, + LabelValues: []string{cs.Name}, + Value: boolFloat64(cs.State.Terminated != nil), + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodInitContainerStatusTerminatedReasonFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_init_container_status_terminated_reason", + "Describes the reason the init container is currently in terminated state.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := make([]*metric.Metric, 0, len(p.Status.InitContainerStatuses)) + for _, cs := range p.Status.InitContainerStatuses { + if cs.State.Terminated != nil { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"container", "reason"}, + LabelValues: []string{cs.Name, cs.State.Terminated.Reason}, + Value: 1, + }) } + } - return &metric.Family{ - Metrics: ms, + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodInitContainerStatusWaitingFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_init_container_status_waiting", + "Describes whether the init container is currently in waiting state.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses)) + + for i, cs := range p.Status.InitContainerStatuses { + ms[i] = &metric.Metric{ + LabelKeys: []string{"container"}, + LabelValues: []string{cs.Name}, + Value: boolFloat64(cs.State.Waiting != nil), + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodInitContainerStatusWaitingReasonFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_init_container_status_waiting_reason", + "Describes the reason the init container is currently in waiting state.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := make([]*metric.Metric, 0, len(p.Status.InitContainerStatuses)) + for _, cs := range p.Status.InitContainerStatuses { + // Skip creating series for running containers. + if cs.State.Waiting != nil { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"container", "reason"}, + LabelValues: []string{cs.Name, cs.State.Waiting.Reason}, + Value: 1, + }) } - }), - }, - { - Name: "kube_pod_status_ready", - Type: metric.Gauge, - Help: "Describes whether the pod is ready to serve requests.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := []*metric.Metric{} - - for _, c := range p.Status.Conditions { - switch c.Type { - case v1.PodReady: - conditionMetrics := addConditionMetrics(c.Status) - - for _, m := range conditionMetrics { - metric := m - metric.LabelKeys = []string{"condition"} - ms = append(ms, metric) - } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodAnnotationsGenerator(allowAnnotations []string) generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_annotations", + "Kubernetes annotations converted to Prometheus labels.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", p.Annotations, allowAnnotations) + m := metric.Metric{ + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + } + return &metric.Family{ + Metrics: []*metric.Metric{&m}, + } + }), + ) +} + +func createPodLabelsGenerator(allowLabelsList []string) generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_labels", + "Kubernetes labels converted to Prometheus labels.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", p.Labels, allowLabelsList) + m := metric.Metric{ + LabelKeys: labelKeys, + LabelValues: labelValues, + Value: 1, + } + return &metric.Family{ + Metrics: []*metric.Metric{&m}, + } + }), + ) +} + +func createPodOverheadCPUCoresFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_overhead_cpu_cores", + "The pod overhead in regards to cpu cores associated with running a pod.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + if p.Spec.Overhead != nil { + for resourceName, val := range p.Spec.Overhead { + if resourceName == v1.ResourceCPU { + ms = append(ms, &metric.Metric{ + Value: float64(val.MilliValue()) / 1000, + }) } } + } + return &metric.Family{ + Metrics: ms, + } + }), + ) +} - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_status_scheduled", - Type: metric.Gauge, - Help: "Describes the status of the scheduling process for the pod.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := []*metric.Metric{} - - for _, c := range p.Status.Conditions { - switch c.Type { - case v1.PodScheduled: - conditionMetrics := addConditionMetrics(c.Status) - - for _, m := range conditionMetrics { - metric := m - metric.LabelKeys = []string{"condition"} - ms = append(ms, metric) - } +func createPodOverheadMemoryBytesFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_overhead_memory_bytes", + "The pod overhead in regards to memory associated with running a pod.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + if p.Spec.Overhead != nil { + for resourceName, val := range p.Spec.Overhead { + if resourceName == v1.ResourceMemory { + ms = append(ms, &metric.Metric{ + Value: float64(val.Value()), + }) } } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} +func createPodOwnerFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_owner", + "Information about the Pod's owner.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + labelKeys := []string{"owner_kind", "owner_name", "owner_is_controller"} + + owners := p.GetOwnerReferences() + if len(owners) == 0 { return &metric.Family{ - Metrics: ms, + Metrics: []*metric.Metric{ + { + LabelKeys: labelKeys, + LabelValues: []string{"", "", ""}, + Value: 1, + }, + }, } - }), - }, - { - Name: "kube_pod_container_info", - Type: metric.Gauge, - Help: "Information about a container in a pod.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := make([]*metric.Metric, len(p.Status.ContainerStatuses)) - labelKeys := []string{"container", "image", "image_id", "container_id"} - - for i, cs := range p.Status.ContainerStatuses { + } + + ms := make([]*metric.Metric, len(owners)) + + for i, owner := range owners { + if owner.Controller != nil { ms[i] = &metric.Metric{ LabelKeys: labelKeys, - LabelValues: []string{cs.Name, cs.Image, cs.ImageID, cs.ContainerID}, + LabelValues: []string{owner.Kind, owner.Name, strconv.FormatBool(*owner.Controller)}, Value: 1, } - } - - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_init_container_info", - Type: metric.Gauge, - Help: "Information about an init container in a pod.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses)) - labelKeys := []string{"container", "image", "image_id", "container_id"} - - for i, cs := range p.Status.InitContainerStatuses { + } else { ms[i] = &metric.Metric{ LabelKeys: labelKeys, - LabelValues: []string{cs.Name, cs.Image, cs.ImageID, cs.ContainerID}, + LabelValues: []string{owner.Kind, owner.Name, "false"}, Value: 1, } } + } - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_container_status_waiting", - Type: metric.Gauge, - Help: "Describes whether the container is currently in waiting state.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := make([]*metric.Metric, len(p.Status.ContainerStatuses)) - - for i, cs := range p.Status.ContainerStatuses { - ms[i] = &metric.Metric{ - LabelKeys: []string{"container"}, - LabelValues: []string{cs.Name}, - Value: boolFloat64(cs.State.Waiting != nil), - } - } + return &metric.Family{ + Metrics: ms, + } + }), + ) +} - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_init_container_status_waiting", - Type: metric.Gauge, - Help: "Describes whether the init container is currently in waiting state.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses)) - - for i, cs := range p.Status.InitContainerStatuses { - ms[i] = &metric.Metric{ - LabelKeys: []string{"container"}, - LabelValues: []string{cs.Name}, - Value: boolFloat64(cs.State.Waiting != nil), - } - } +func createPodRestartPolicyFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_restart_policy", + "Describes the restart policy in use by this pod.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: []string{"type"}, + LabelValues: []string{string(p.Spec.RestartPolicy)}, + Value: float64(1), + }, + }, + } + }), + ) +} - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_container_status_waiting_reason", - Type: metric.Gauge, - Help: "Describes the reason the container is currently in waiting state.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := make([]*metric.Metric, len(p.Status.ContainerStatuses)*len(containerWaitingReasons)) - - for i, cs := range p.Status.ContainerStatuses { - for j, reason := range containerWaitingReasons { - ms[i*len(containerWaitingReasons)+j] = &metric.Metric{ - LabelKeys: []string{"container", "reason"}, - LabelValues: []string{cs.Name, reason}, - Value: boolFloat64(waitingReason(cs, reason)), - } - } - } +func createPodRuntimeClassNameInfoFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_runtimeclass_name_info", + "The runtimeclass associated with the pod.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + if p.Spec.RuntimeClassName != nil { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"runtimeclass_name"}, + LabelValues: []string{*p.Spec.RuntimeClassName}, + Value: 1, + }) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_init_container_status_waiting_reason", - Type: metric.Gauge, - Help: "Describes the reason the init container is currently in waiting state.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses)*len(containerWaitingReasons)) - - for i, cs := range p.Status.InitContainerStatuses { - for j, reason := range containerWaitingReasons { - ms[i*len(containerWaitingReasons)+j] = &metric.Metric{ - LabelKeys: []string{"container", "reason"}, - LabelValues: []string{cs.Name, reason}, - Value: boolFloat64(waitingReason(cs, reason)), - } - } +func createPodSpecVolumesPersistentVolumeClaimsInfoFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_spec_volumes_persistentvolumeclaims_info", + "Information about persistentvolumeclaim volumes in a pod.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + for _, v := range p.Spec.Volumes { + if v.PersistentVolumeClaim != nil { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"volume", "persistentvolumeclaim"}, + LabelValues: []string{v.Name, v.PersistentVolumeClaim.ClaimName}, + Value: 1, + }) } + } - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_container_status_running", - Type: metric.Gauge, - Help: "Describes whether the container is currently in running state.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := make([]*metric.Metric, len(p.Status.ContainerStatuses)) - - for i, cs := range p.Status.ContainerStatuses { - ms[i] = &metric.Metric{ - LabelKeys: []string{"container"}, - LabelValues: []string{cs.Name}, - Value: boolFloat64(cs.State.Running != nil), - } - } + return &metric.Family{ + Metrics: ms, + } + }), + ) +} - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_init_container_status_running", - Type: metric.Gauge, - Help: "Describes whether the init container is currently in running state.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses)) - - for i, cs := range p.Status.InitContainerStatuses { - ms[i] = &metric.Metric{ - LabelKeys: []string{"container"}, - LabelValues: []string{cs.Name}, - Value: boolFloat64(cs.State.Running != nil), - } +func createPodSpecVolumesPersistentVolumeClaimsReadonlyFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_spec_volumes_persistentvolumeclaims_readonly", + "Describes whether a persistentvolumeclaim is mounted read only.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + for _, v := range p.Spec.Volumes { + if v.PersistentVolumeClaim != nil { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"volume", "persistentvolumeclaim"}, + LabelValues: []string{v.Name, v.PersistentVolumeClaim.ClaimName}, + Value: boolFloat64(v.PersistentVolumeClaim.ReadOnly), + }) } + } - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_container_status_terminated", - Type: metric.Gauge, - Help: "Describes whether the container is currently in terminated state.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := make([]*metric.Metric, len(p.Status.ContainerStatuses)) - - for i, cs := range p.Status.ContainerStatuses { - ms[i] = &metric.Metric{ - LabelKeys: []string{"container"}, - LabelValues: []string{cs.Name}, - Value: boolFloat64(cs.State.Terminated != nil), - } - } + return &metric.Family{ + Metrics: ms, + } + }), + ) +} - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_init_container_status_terminated", - Type: metric.Gauge, - Help: "Describes whether the init container is currently in terminated state.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses)) - - for i, cs := range p.Status.InitContainerStatuses { - ms[i] = &metric.Metric{ - LabelKeys: []string{"container"}, - LabelValues: []string{cs.Name}, - Value: boolFloat64(cs.State.Terminated != nil), - } - } +func createPodStartTimeFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_start_time", + "Start time in unix timestamp for a pod.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + if p.Status.StartTime != nil { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64((p.Status.StartTime).Unix()), + }) + } + return &metric.Family{ + Metrics: ms, + } + }), + ) +} +func createPodStatusPhaseFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_status_phase", + "The pods current phase.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + phase := p.Status.Phase + if phase == "" { return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_container_status_terminated_reason", - Type: metric.Gauge, - Help: "Describes the reason the container is currently in terminated state.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := make([]*metric.Metric, len(p.Status.ContainerStatuses)*len(containerTerminatedReasons)) - - for i, cs := range p.Status.ContainerStatuses { - for j, reason := range containerTerminatedReasons { - ms[i*len(containerTerminatedReasons)+j] = &metric.Metric{ - LabelKeys: []string{"container", "reason"}, - LabelValues: []string{cs.Name, reason}, - Value: boolFloat64(terminationReason(cs, reason)), - } - } + Metrics: []*metric.Metric{}, } + } - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_init_container_status_terminated_reason", - Type: metric.Gauge, - Help: "Describes the reason the init container is currently in terminated state.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses)*len(containerTerminatedReasons)) - - for i, cs := range p.Status.InitContainerStatuses { - for j, reason := range containerTerminatedReasons { - ms[i*len(containerTerminatedReasons)+j] = &metric.Metric{ - LabelKeys: []string{"container", "reason"}, - LabelValues: []string{cs.Name, reason}, - Value: boolFloat64(terminationReason(cs, reason)), - } - } - } + phases := []struct { + v bool + n string + }{ + {phase == v1.PodPending, string(v1.PodPending)}, + {phase == v1.PodSucceeded, string(v1.PodSucceeded)}, + {phase == v1.PodFailed, string(v1.PodFailed)}, + {phase == v1.PodUnknown, string(v1.PodUnknown)}, + {phase == v1.PodRunning, string(v1.PodRunning)}, + } - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_container_status_last_terminated_reason", - Type: metric.Gauge, - Help: "Describes the last reason the container was in terminated state.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := make([]*metric.Metric, len(p.Status.ContainerStatuses)*len(containerTerminatedReasons)) - - for i, cs := range p.Status.ContainerStatuses { - for j, reason := range containerTerminatedReasons { - ms[i*len(containerTerminatedReasons)+j] = &metric.Metric{ - LabelKeys: []string{"container", "reason"}, - LabelValues: []string{cs.Name, reason}, - Value: boolFloat64(lastTerminationReason(cs, reason)), - } - } - } + ms := make([]*metric.Metric, len(phases)) - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_init_container_status_last_terminated_reason", - Type: metric.Gauge, - Help: "Describes the last reason the init container was in terminated state.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses)*len(containerTerminatedReasons)) - - for i, cs := range p.Status.InitContainerStatuses { - for j, reason := range containerTerminatedReasons { - ms[i*len(containerTerminatedReasons)+j] = &metric.Metric{ - LabelKeys: []string{"container", "reason"}, - LabelValues: []string{cs.Name, reason}, - Value: boolFloat64(lastTerminationReason(cs, reason)), - } - } - } + for i, p := range phases { + ms[i] = &metric.Metric{ - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_container_status_ready", - Type: metric.Gauge, - Help: "Describes whether the containers readiness check succeeded.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := make([]*metric.Metric, len(p.Status.ContainerStatuses)) - - for i, cs := range p.Status.ContainerStatuses { - ms[i] = &metric.Metric{ - LabelKeys: []string{"container"}, - LabelValues: []string{cs.Name}, - Value: boolFloat64(cs.Ready), - } + LabelKeys: []string{"phase"}, + LabelValues: []string{p.n}, + Value: boolFloat64(p.v), } + } - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_init_container_status_ready", - Type: metric.Gauge, - Help: "Describes whether the init containers readiness check succeeded.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses)) - - for i, cs := range p.Status.InitContainerStatuses { - ms[i] = &metric.Metric{ - LabelKeys: []string{"container"}, - LabelValues: []string{cs.Name}, - Value: boolFloat64(cs.Ready), - } - } + return &metric.Family{ + Metrics: ms, + } + }), + ) +} - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_container_status_restarts_total", - Type: metric.Counter, - Help: "The number of container restarts per container.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := make([]*metric.Metric, len(p.Status.ContainerStatuses)) - - for i, cs := range p.Status.ContainerStatuses { - ms[i] = &metric.Metric{ - LabelKeys: []string{"container"}, - LabelValues: []string{cs.Name}, - Value: float64(cs.RestartCount), - } +func createPodStatusContainerReadyTimeFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_status_container_ready_time", + "Readiness achieved time in unix timestamp for a pod containers.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + for _, c := range p.Status.Conditions { + if c.Type == v1.ContainersReady && c.Status == v1.ConditionTrue { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64((c.LastTransitionTime).Unix()), + }) } + } - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_init_container_status_restarts_total", - Type: metric.Counter, - Help: "The number of restarts for the init container.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := make([]*metric.Metric, len(p.Status.InitContainerStatuses)) - - for i, cs := range p.Status.InitContainerStatuses { - ms[i] = &metric.Metric{ - LabelKeys: []string{"container"}, - LabelValues: []string{cs.Name}, - Value: float64(cs.RestartCount), - } - } + return &metric.Family{ + Metrics: ms, + } + }), + ) +} - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_container_resource_requests", - Type: metric.Gauge, - Help: "The number of requested request resource by a container.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := []*metric.Metric{} - - for _, c := range p.Spec.Containers { - req := c.Resources.Requests - - for resourceName, val := range req { - switch resourceName { - case v1.ResourceCPU: - ms = append(ms, &metric.Metric{ - LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitCore)}, - Value: float64(val.MilliValue()) / 1000, - }) - case v1.ResourceStorage: - fallthrough - case v1.ResourceEphemeralStorage: - fallthrough - case v1.ResourceMemory: - ms = append(ms, &metric.Metric{ - LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, - Value: float64(val.Value()), - }) - default: - if isHugePageResourceName(resourceName) { - ms = append(ms, &metric.Metric{ - LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, - Value: float64(val.Value()), - }) - } - if isAttachableVolumeResourceName(resourceName) { - ms = append(ms, &metric.Metric{ - LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, - Value: float64(val.Value()), - }) - } - if isExtendedResourceName(resourceName) { - ms = append(ms, &metric.Metric{ - LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitInteger)}, - Value: float64(val.Value()), - }) - } - } - } - } +func createPodStatusReadyTimeFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_status_ready_time", + "Readiness achieved time in unix timestamp for a pod.", + metric.Gauge, + basemetrics.ALPHA, - for _, metric := range ms { - metric.LabelKeys = []string{"container", "node", "resource", "unit"} - } + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_container_resource_limits", - Type: metric.Gauge, - Help: "The number of requested limit resource by a container.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := []*metric.Metric{} - - for _, c := range p.Spec.Containers { - lim := c.Resources.Limits - - for resourceName, val := range lim { - switch resourceName { - case v1.ResourceCPU: - ms = append(ms, &metric.Metric{ - Value: float64(val.MilliValue()) / 1000, - LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitCore)}, - }) - case v1.ResourceStorage: - fallthrough - case v1.ResourceEphemeralStorage: - fallthrough - case v1.ResourceMemory: - ms = append(ms, &metric.Metric{ - LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, - Value: float64(val.Value()), - }) - default: - if isHugePageResourceName(resourceName) { - ms = append(ms, &metric.Metric{ - LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, - Value: float64(val.Value()), - }) - } - if isAttachableVolumeResourceName(resourceName) { - ms = append(ms, &metric.Metric{ - Value: float64(val.Value()), - LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, - }) - } - if isExtendedResourceName(resourceName) { - ms = append(ms, &metric.Metric{ - Value: float64(val.Value()), - LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitInteger)}, - }) - } - } - } + for _, c := range p.Status.Conditions { + if c.Type == v1.PodReady && c.Status == v1.ConditionTrue { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64((c.LastTransitionTime).Unix()), + }) } + } - for _, metric := range ms { - metric.LabelKeys = []string{"container", "node", "resource", "unit"} - } + return &metric.Family{ + Metrics: ms, + } + }), + ) +} +func createPodStatusQosClassFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_status_qos_class", + "The pods current qosClass.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + class := p.Status.QOSClass + if class == "" { return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_init_container_resource_limits", - Type: metric.Gauge, - Help: "The number of requested limit resource by the init container.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := []*metric.Metric{} - - for _, c := range p.Spec.InitContainers { - lim := c.Resources.Limits - - for resourceName, val := range lim { - switch resourceName { - case v1.ResourceCPU: - ms = append(ms, &metric.Metric{ - Value: float64(val.MilliValue()) / 1000, - LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitCore)}, - }) - case v1.ResourceStorage: - fallthrough - case v1.ResourceEphemeralStorage: - fallthrough - case v1.ResourceMemory: - ms = append(ms, &metric.Metric{ - LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, - Value: float64(val.Value()), - }) - default: - if isHugePageResourceName(resourceName) { - ms = append(ms, &metric.Metric{ - LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, - Value: float64(val.Value()), - }) - } - if isAttachableVolumeResourceName(resourceName) { - ms = append(ms, &metric.Metric{ - Value: float64(val.Value()), - LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, - }) - } - if isExtendedResourceName(resourceName) { - ms = append(ms, &metric.Metric{ - Value: float64(val.Value()), - LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitInteger)}, - }) - } - } - } + Metrics: []*metric.Metric{}, } + } - for _, metric := range ms { - metric.LabelKeys = []string{"container", "node", "resource", "unit"} - } + qosClasses := []struct { + v bool + n string + }{ + {class == v1.PodQOSBestEffort, string(v1.PodQOSBestEffort)}, + {class == v1.PodQOSBurstable, string(v1.PodQOSBurstable)}, + {class == v1.PodQOSGuaranteed, string(v1.PodQOSGuaranteed)}, + } - return &metric.Family{ - Metrics: ms, + ms := make([]*metric.Metric, len(qosClasses)) + + for i, p := range qosClasses { + ms[i] = &metric.Metric{ + + LabelKeys: []string{"qos_class"}, + LabelValues: []string{p.n}, + Value: boolFloat64(p.v), } - }), - }, - { - Name: "kube_pod_container_resource_requests_cpu_cores", - Type: metric.Gauge, - Help: "The number of requested cpu cores by a container.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := []*metric.Metric{} - - for _, c := range p.Spec.Containers { - req := c.Resources.Requests - if cpu, ok := req[v1.ResourceCPU]; ok { - ms = append(ms, &metric.Metric{ - LabelKeys: []string{"container", "node"}, - LabelValues: []string{c.Name, p.Spec.NodeName}, - Value: float64(cpu.MilliValue()) / 1000, - }) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodStatusReadyFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_status_ready", + "Describes whether the pod is ready to serve requests.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + for _, c := range p.Status.Conditions { + if c.Type == v1.PodReady { + conditionMetrics := addConditionMetrics(c.Status) + + for _, m := range conditionMetrics { + metric := m + metric.LabelKeys = []string{"condition"} + ms = append(ms, metric) } } + } - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_container_resource_requests_memory_bytes", - Type: metric.Gauge, - Help: "The number of requested memory bytes by a container.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := []*metric.Metric{} - - for _, c := range p.Spec.Containers { - req := c.Resources.Requests - if mem, ok := req[v1.ResourceMemory]; ok { - ms = append(ms, &metric.Metric{ - LabelKeys: []string{"container", "node"}, - LabelValues: []string{c.Name, p.Spec.NodeName}, - Value: float64(mem.Value()), - }) + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodStatusReasonFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_status_reason", + "The pod status reasons", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + for _, reason := range podStatusReasons { + metric := &metric.Metric{} + metric.LabelKeys = []string{"reason"} + metric.LabelValues = []string{reason} + if p.Status.Reason == reason { + metric.Value = boolFloat64(true) + } else { + metric.Value = boolFloat64(false) + } + ms = append(ms, metric) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodStatusScheduledFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_status_scheduled", + "Describes the status of the scheduling process for the pod.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + for _, c := range p.Status.Conditions { + if c.Type == v1.PodScheduled { + conditionMetrics := addConditionMetrics(c.Status) + + for _, m := range conditionMetrics { + metric := m + metric.LabelKeys = []string{"condition"} + ms = append(ms, metric) } } + } - return &metric.Family{ - Metrics: ms, - } - }), - }, - { - Name: "kube_pod_container_resource_limits_cpu_cores", - Type: metric.Gauge, - Help: "The limit on cpu cores to be used by a container.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := []*metric.Metric{} - - for _, c := range p.Spec.Containers { - lim := c.Resources.Limits - if cpu, ok := lim[v1.ResourceCPU]; ok { - ms = append(ms, &metric.Metric{ - LabelKeys: []string{"container", "node"}, - LabelValues: []string{c.Name, p.Spec.NodeName}, - Value: float64(cpu.MilliValue()) / 1000, - }) - } + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodStatusScheduledTimeFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_status_scheduled_time", + "Unix timestamp when pod moved into scheduled status", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + for _, c := range p.Status.Conditions { + if c.Type == v1.PodScheduled && c.Status == v1.ConditionTrue { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64(c.LastTransitionTime.Unix()), + }) } + } - return &metric.Family{ - Metrics: ms, + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodStatusUnschedulableFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_status_unschedulable", + "Describes the unschedulable status for the pod.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + + for _, c := range p.Status.Conditions { + if c.Type == v1.PodScheduled && c.Status == v1.ConditionFalse { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: 1, + }) } - }), - }, - { - Name: "kube_pod_container_resource_limits_memory_bytes", - Type: metric.Gauge, - Help: "The limit on memory to be used by a container in bytes.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := []*metric.Metric{} + } - for _, c := range p.Spec.Containers { - lim := c.Resources.Limits + return &metric.Family{ + Metrics: ms, + } + }), + ) +} - if mem, ok := lim[v1.ResourceMemory]; ok { - ms = append(ms, &metric.Metric{ - LabelKeys: []string{"container", "node"}, - LabelValues: []string{c.Name, p.Spec.NodeName}, - Value: float64(mem.Value()), - }) - } +func createPodTolerationsFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_tolerations", + "Information about the pod tolerations", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + var ms []*metric.Metric + + for _, t := range p.Spec.Tolerations { + var labelKeys []string + var labelValues []string + + if t.Key != "" { + labelKeys = append(labelKeys, "key") + labelValues = append(labelValues, t.Key) } - return &metric.Family{ - Metrics: ms, + if t.Operator != "" { + labelKeys = append(labelKeys, "operator") + labelValues = append(labelValues, string(t.Operator)) } - }), - }, - { - Name: "kube_pod_spec_volumes_persistentvolumeclaims_info", - Type: metric.Gauge, - Help: "Information about persistentvolumeclaim volumes in a pod.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := []*metric.Metric{} - - for _, v := range p.Spec.Volumes { - if v.PersistentVolumeClaim != nil { - ms = append(ms, &metric.Metric{ - LabelKeys: []string{"volume", "persistentvolumeclaim"}, - LabelValues: []string{v.Name, v.PersistentVolumeClaim.ClaimName}, - Value: 1, - }) - } + + if t.Value != "" { + labelKeys = append(labelKeys, "value") + labelValues = append(labelValues, t.Value) } - return &metric.Family{ - Metrics: ms, + if t.Effect != "" { + labelKeys = append(labelKeys, "effect") + labelValues = append(labelValues, string(t.Effect)) } - }), - }, - { - Name: "kube_pod_spec_volumes_persistentvolumeclaims_readonly", - Type: metric.Gauge, - Help: "Describes whether a persistentvolumeclaim is mounted read only.", - GenerateFunc: wrapPodFunc(func(p *v1.Pod) *metric.Family { - ms := []*metric.Metric{} - - for _, v := range p.Spec.Volumes { - if v.PersistentVolumeClaim != nil { - ms = append(ms, &metric.Metric{ - LabelKeys: []string{"volume", "persistentvolumeclaim"}, - LabelValues: []string{v.Name, v.PersistentVolumeClaim.ClaimName}, - Value: boolFloat64(v.PersistentVolumeClaim.ReadOnly), - }) - } + + if t.TolerationSeconds != nil { + labelKeys = append(labelKeys, "toleration_seconds") + labelValues = append(labelValues, strconv.FormatInt(*t.TolerationSeconds, 10)) } - return &metric.Family{ - Metrics: ms, + if len(labelKeys) == 0 { + continue } - }), - }, - } -) + + ms = append(ms, &metric.Metric{ + LabelKeys: labelKeys, + LabelValues: labelValues, + Value: 1, + }) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createPodNodeSelectorsFamilyGenerator() generator.FamilyGenerator { + return *generator.NewOptInFamilyGenerator( + "kube_pod_nodeselectors", + "Describes the Pod nodeSelectors.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + labelKeys, labelValues := kubeMapToPrometheusLabels("nodeselector", p.Spec.NodeSelector) + m := metric.Metric{ + LabelKeys: labelKeys, + LabelValues: labelValues, + Value: 1, + } + return &metric.Family{ + Metrics: []*metric.Metric{&m}, + } + }), + ) +} func wrapPodFunc(f func(*v1.Pod) *metric.Family) func(interface{}) *metric.Family { return func(obj interface{}) *metric.Family { @@ -1048,42 +1659,22 @@ func wrapPodFunc(f func(*v1.Pod) *metric.Family) func(interface{}) *metric.Famil metricFamily := f(pod) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descPodLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{pod.Namespace, pod.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descPodLabelsDefaultLabels, []string{pod.Namespace, pod.Name, string(pod.UID)}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createPodListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createPodListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.CoreV1().Pods(ns).List(opts) + opts.FieldSelector = fieldSelector + return kubeClient.CoreV1().Pods(ns).List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.CoreV1().Pods(ns).Watch(opts) + opts.FieldSelector = fieldSelector + return kubeClient.CoreV1().Pods(ns).Watch(context.TODO(), opts) }, } } - -func waitingReason(cs v1.ContainerStatus, reason string) bool { - if cs.State.Waiting == nil { - return false - } - return cs.State.Waiting.Reason == reason -} - -func terminationReason(cs v1.ContainerStatus, reason string) bool { - if cs.State.Terminated == nil { - return false - } - return cs.State.Terminated.Reason == reason -} - -func lastTerminationReason(cs v1.ContainerStatus, reason string) bool { - if cs.LastTerminationState.Terminated == nil { - return false - } - return cs.LastTerminationState.Terminated.Reason == reason -} diff --git a/internal/store/pod_test.go b/internal/store/pod_test.go index a09468557a..c66467d937 100644 --- a/internal/store/pod_test.go +++ b/internal/store/pod_test.go @@ -24,11 +24,13 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" + "k8s.io/kube-state-metrics/v2/pkg/options" ) func TestPodStore(t *testing.T) { var test = true + runtimeclass := "foo" startTime := 1501569018 metav1StartTime := metav1.Unix(int64(startTime), 0) @@ -38,6 +40,15 @@ func TestPodStore(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "pod1", Namespace: "ns1", + UID: "uid1", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container1", + Image: "k8s.gcr.io/hyperkube1_spec", + }, + }, }, Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ @@ -51,9 +62,9 @@ func TestPodStore(t *testing.T) { }, }, Want: ` - # HELP kube_pod_container_info Information about a container in a pod. + # HELP kube_pod_container_info [STABLE] Information about a container in a pod. # TYPE kube_pod_container_info gauge - kube_pod_container_info{container="container1",container_id="docker://ab123",image="k8s.gcr.io/hyperkube1",image_id="docker://sha256:aaa",namespace="ns1",pod="pod1"} 1`, + kube_pod_container_info{container="container1",container_id="docker://ab123",image="k8s.gcr.io/hyperkube1",image_spec="k8s.gcr.io/hyperkube1_spec",image_id="docker://sha256:aaa",namespace="ns1",pod="pod1",uid="uid1"} 1`, MetricNames: []string{"kube_pod_container_info"}, }, { @@ -61,6 +72,25 @@ func TestPodStore(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "pod2", Namespace: "ns2", + UID: "uid2", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container2", + Image: "k8s.gcr.io/hyperkube2_spec", + }, + { + Name: "container3", + Image: "k8s.gcr.io/hyperkube3_spec", + }, + }, + InitContainers: []v1.Container{ + { + Name: "initContainer", + Image: "k8s.gcr.io/initfoo_spec", + }, + }, }, Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ @@ -88,13 +118,13 @@ func TestPodStore(t *testing.T) { }, }, Want: ` - # HELP kube_pod_container_info Information about a container in a pod. - # HELP kube_pod_init_container_info Information about an init container in a pod. + # HELP kube_pod_container_info [STABLE] Information about a container in a pod. + # HELP kube_pod_init_container_info [STABLE] Information about an init container in a pod. # TYPE kube_pod_container_info gauge # TYPE kube_pod_init_container_info gauge - kube_pod_container_info{container="container2",container_id="docker://cd456",image="k8s.gcr.io/hyperkube2",image_id="docker://sha256:bbb",namespace="ns2",pod="pod2"} 1 - kube_pod_container_info{container="container3",container_id="docker://ef789",image="k8s.gcr.io/hyperkube3",image_id="docker://sha256:ccc",namespace="ns2",pod="pod2"} 1 - kube_pod_init_container_info{container="initContainer",container_id="docker://ef123",image="k8s.gcr.io/initfoo",image_id="docker://sha256:wxyz",namespace="ns2",pod="pod2"} 1`, + kube_pod_container_info{container="container2",container_id="docker://cd456",image_spec="k8s.gcr.io/hyperkube2_spec",image="k8s.gcr.io/hyperkube2",image_id="docker://sha256:bbb",namespace="ns2",pod="pod2",uid="uid2"} 1 + kube_pod_container_info{container="container3",container_id="docker://ef789",image_spec="k8s.gcr.io/hyperkube3_spec",image="k8s.gcr.io/hyperkube3",image_id="docker://sha256:ccc",namespace="ns2",pod="pod2",uid="uid2"} 1 + kube_pod_init_container_info{container="initContainer",container_id="docker://ef123",image_spec="k8s.gcr.io/initfoo_spec",image="k8s.gcr.io/initfoo",image_id="docker://sha256:wxyz",namespace="ns2",pod="pod2",uid="uid2"} 1`, MetricNames: []string{"kube_pod_container_info", "kube_pod_init_container_info"}, }, { @@ -102,6 +132,15 @@ func TestPodStore(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "pod1", Namespace: "ns1", + UID: "uid1", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container1", + Image: "k8s.gcr.io/hyperkube1_spec", + }, + }, }, Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ @@ -113,9 +152,9 @@ func TestPodStore(t *testing.T) { }, }, Want: ` - # HELP kube_pod_container_status_ready Describes whether the containers readiness check succeeded. + # HELP kube_pod_container_status_ready [STABLE] Describes whether the containers readiness check succeeded. # TYPE kube_pod_container_status_ready gauge - kube_pod_container_status_ready{container="container1",namespace="ns1",pod="pod1"} 1`, + kube_pod_container_status_ready{container="container1",namespace="ns1",pod="pod1",uid="uid1"} 1`, MetricNames: []string{"kube_pod_container_status_ready"}, }, { @@ -123,6 +162,19 @@ func TestPodStore(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "pod2", Namespace: "ns2", + UID: "uid2", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container2", + Image: "k8s.gcr.io/hyperkube2_spec", + }, + { + Name: "container3", + Image: "k8s.gcr.io/hyperkube3_spec", + }, + }, }, Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ @@ -138,10 +190,10 @@ func TestPodStore(t *testing.T) { }, }, Want: ` - # HELP kube_pod_container_status_ready Describes whether the containers readiness check succeeded. + # HELP kube_pod_container_status_ready [STABLE] Describes whether the containers readiness check succeeded. # TYPE kube_pod_container_status_ready gauge - kube_pod_container_status_ready{container="container2",namespace="ns2",pod="pod2"} 1 - kube_pod_container_status_ready{container="container3",namespace="ns2",pod="pod2"} 0 + kube_pod_container_status_ready{container="container2",namespace="ns2",pod="pod2",uid="uid2"} 1 + kube_pod_container_status_ready{container="container3",namespace="ns2",pod="pod2",uid="uid2"} 0 `, MetricNames: []string{"kube_pod_container_status_ready"}, }, @@ -150,6 +202,29 @@ func TestPodStore(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "pod3", Namespace: "ns3", + UID: "uid3", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container2", + Image: "k8s.gcr.io/hyperkube2_spec", + }, + { + Name: "container3", + Image: "k8s.gcr.io/hyperkube3_spec", + }, + }, + InitContainers: []v1.Container{ + { + Name: "initcontainer1", + Image: "k8s.gcr.io/initfoo_spec", + }, + { + Name: "initcontainer2", + Image: "k8s.gcr.io/initfoo_spec", + }, + }, }, Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ @@ -175,10 +250,10 @@ func TestPodStore(t *testing.T) { }, }, Want: ` - # HELP kube_pod_init_container_status_ready Describes whether the init containers readiness check succeeded. + # HELP kube_pod_init_container_status_ready [STABLE] Describes whether the init containers readiness check succeeded. # TYPE kube_pod_init_container_status_ready gauge - kube_pod_init_container_status_ready{container="initcontainer1",namespace="ns3",pod="pod3"} 1 - kube_pod_init_container_status_ready{container="initcontainer2",namespace="ns3",pod="pod3"} 0 + kube_pod_init_container_status_ready{container="initcontainer1",namespace="ns3",pod="pod3",uid="uid3"} 1 + kube_pod_init_container_status_ready{container="initcontainer2",namespace="ns3",pod="pod3",uid="uid3"} 0 `, MetricNames: []string{"kube_pod_init_container_status_ready"}, }, @@ -187,6 +262,15 @@ func TestPodStore(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "pod1", Namespace: "ns1", + UID: "uid1", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container1", + Image: "k8s.gcr.io/hyperkube1_spec", + }, + }, }, Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ @@ -198,9 +282,9 @@ func TestPodStore(t *testing.T) { }, }, Want: ` - # HELP kube_pod_container_status_restarts_total The number of container restarts per container. + # HELP kube_pod_container_status_restarts_total [STABLE] The number of container restarts per container. # TYPE kube_pod_container_status_restarts_total counter - kube_pod_container_status_restarts_total{container="container1",namespace="ns1",pod="pod1"} 0 + kube_pod_container_status_restarts_total{container="container1",namespace="ns1",pod="pod1",uid="uid1"} 0 `, MetricNames: []string{"kube_pod_container_status_restarts_total"}, }, @@ -209,6 +293,15 @@ func TestPodStore(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "pod2", Namespace: "ns2", + UID: "uid2", + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Name: "initcontainer1", + Image: "k8s.gcr.io/initfoo_spec", + }, + }, }, Status: v1.PodStatus{ InitContainerStatuses: []v1.ContainerStatus{ @@ -220,9 +313,9 @@ func TestPodStore(t *testing.T) { }, }, Want: ` - # HELP kube_pod_init_container_status_restarts_total The number of restarts for the init container. + # HELP kube_pod_init_container_status_restarts_total [STABLE] The number of restarts for the init container. # TYPE kube_pod_init_container_status_restarts_total counter - kube_pod_init_container_status_restarts_total{container="initcontainer1",namespace="ns2",pod="pod2"} 1 + kube_pod_init_container_status_restarts_total{container="initcontainer1",namespace="ns2",pod="pod2",uid="uid2"} 1 `, MetricNames: []string{"kube_pod_init_container_status_restarts_total"}, }, @@ -231,6 +324,19 @@ func TestPodStore(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "pod2", Namespace: "ns2", + UID: "uid2", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container2", + Image: "k8s.gcr.io/hyperkube2_spec", + }, + { + Name: "container3", + Image: "k8s.gcr.io/hyperkube3_spec", + }, + }, }, Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ @@ -246,10 +352,10 @@ func TestPodStore(t *testing.T) { }, }, Want: ` - # HELP kube_pod_container_status_restarts_total The number of container restarts per container. + # HELP kube_pod_container_status_restarts_total [STABLE] The number of container restarts per container. # TYPE kube_pod_container_status_restarts_total counter - kube_pod_container_status_restarts_total{container="container2",namespace="ns2",pod="pod2"} 0 - kube_pod_container_status_restarts_total{container="container3",namespace="ns2",pod="pod2"} 1 + kube_pod_container_status_restarts_total{container="container2",namespace="ns2",pod="pod2",uid="uid2"} 0 + kube_pod_container_status_restarts_total{container="container3",namespace="ns2",pod="pod2",uid="uid2"} 1 `, MetricNames: []string{"kube_pod_container_status_restarts_total"}, }, @@ -258,6 +364,19 @@ func TestPodStore(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "pod2", Namespace: "ns2", + UID: "uid2", + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Name: "initcontainer2", + Image: "k8s.gcr.io/initfoo_spec", + }, + { + Name: "initcontainer3", + Image: "k8s.gcr.io/initfoo_spec", + }, + }, }, Status: v1.PodStatus{ InitContainerStatuses: []v1.ContainerStatus{ @@ -273,10 +392,10 @@ func TestPodStore(t *testing.T) { }, }, Want: ` - # HELP kube_pod_init_container_status_restarts_total The number of restarts for the init container. + # HELP kube_pod_init_container_status_restarts_total [STABLE] The number of restarts for the init container. # TYPE kube_pod_init_container_status_restarts_total counter - kube_pod_init_container_status_restarts_total{container="initcontainer2",namespace="ns2",pod="pod2"} 0 - kube_pod_init_container_status_restarts_total{container="initcontainer3",namespace="ns2",pod="pod2"} 1 + kube_pod_init_container_status_restarts_total{container="initcontainer2",namespace="ns2",pod="pod2",uid="uid2"} 0 + kube_pod_init_container_status_restarts_total{container="initcontainer3",namespace="ns2",pod="pod2",uid="uid2"} 1 `, MetricNames: []string{"kube_pod_init_container_status_restarts_total"}, }, @@ -285,13 +404,32 @@ func TestPodStore(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "pod1", Namespace: "ns1", + UID: "uid1", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container1", + Image: "k8s.gcr.io/hyperkube2_spec", + }, + }, + InitContainers: []v1.Container{ + { + Name: "initcontainer1", + Image: "k8s.gcr.io/initfoo_spec", + }, + }, }, Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ { Name: "container1", State: v1.ContainerState{ - Running: &v1.ContainerStateRunning{}, + Running: &v1.ContainerStateRunning{ + StartedAt: metav1.Time{ + Time: time.Unix(1501777018, 0), + }, + }, }, }, }, @@ -306,17 +444,19 @@ func TestPodStore(t *testing.T) { }, }, Want: ` - # HELP kube_pod_container_status_running Describes whether the container is currently in running state. - # HELP kube_pod_container_status_terminated Describes whether the container is currently in terminated state. + # HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state. + # HELP kube_pod_container_state_started [STABLE] Start time in unix timestamp for a pod container. + # HELP kube_pod_container_status_terminated [STABLE] Describes whether the container is currently in terminated state. # HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state. - # HELP kube_pod_container_status_waiting Describes whether the container is currently in waiting state. - # HELP kube_pod_container_status_waiting_reason Describes the reason the container is currently in waiting state. - # HELP kube_pod_init_container_status_running Describes whether the init container is currently in running state. - # HELP kube_pod_init_container_status_terminated Describes whether the init container is currently in terminated state. + # HELP kube_pod_container_status_waiting [STABLE] Describes whether the container is currently in waiting state. + # HELP kube_pod_container_status_waiting_reason [STABLE] Describes the reason the container is currently in waiting state. + # HELP kube_pod_init_container_status_running [STABLE] Describes whether the init container is currently in running state. + # HELP kube_pod_init_container_status_terminated [STABLE] Describes whether the init container is currently in terminated state. # HELP kube_pod_init_container_status_terminated_reason Describes the reason the init container is currently in terminated state. - # HELP kube_pod_init_container_status_waiting Describes whether the init container is currently in waiting state. + # HELP kube_pod_init_container_status_waiting [STABLE] Describes whether the init container is currently in waiting state. # HELP kube_pod_init_container_status_waiting_reason Describes the reason the init container is currently in waiting state. # TYPE kube_pod_container_status_running gauge + # TYPE kube_pod_container_state_started gauge # TYPE kube_pod_container_status_terminated gauge # TYPE kube_pod_container_status_terminated_reason gauge # TYPE kube_pod_container_status_waiting gauge @@ -326,43 +466,18 @@ func TestPodStore(t *testing.T) { # TYPE kube_pod_init_container_status_terminated_reason gauge # TYPE kube_pod_init_container_status_waiting gauge # TYPE kube_pod_init_container_status_waiting_reason gauge - kube_pod_container_status_running{container="container1",namespace="ns1",pod="pod1"} 1 - kube_pod_container_status_terminated_reason{container="container1",namespace="ns1",pod="pod1",reason="Completed"} 0 - kube_pod_container_status_terminated_reason{container="container1",namespace="ns1",pod="pod1",reason="ContainerCannotRun"} 0 - kube_pod_container_status_terminated_reason{container="container1",namespace="ns1",pod="pod1",reason="Error"} 0 - kube_pod_container_status_terminated_reason{container="container1",namespace="ns1",pod="pod1",reason="Evicted"} 0 - kube_pod_container_status_terminated_reason{container="container1",namespace="ns1",pod="pod1",reason="OOMKilled"} 0 - kube_pod_container_status_terminated_reason{container="container1",namespace="ns1",pod="pod1",reason="DeadlineExceeded"} 0 - kube_pod_container_status_terminated{container="container1",namespace="ns1",pod="pod1"} 0 - kube_pod_container_status_waiting{container="container1",namespace="ns1",pod="pod1"} 0 - kube_pod_container_status_waiting_reason{container="container1",namespace="ns1",pod="pod1",reason="ContainerCreating"} 0 - kube_pod_container_status_waiting_reason{container="container1",namespace="ns1",pod="pod1",reason="ImagePullBackOff"} 0 - kube_pod_container_status_waiting_reason{container="container1",namespace="ns1",pod="pod1",reason="CrashLoopBackOff"} 0 - kube_pod_container_status_waiting_reason{container="container1",namespace="ns1",pod="pod1",reason="ErrImagePull"} 0 - kube_pod_container_status_waiting_reason{container="container1",namespace="ns1",pod="pod1",reason="CreateContainerConfigError"} 0 - kube_pod_container_status_waiting_reason{container="container1",namespace="ns1",pod="pod1",reason="CreateContainerError"} 0 - kube_pod_container_status_waiting_reason{container="container1",namespace="ns1",pod="pod1",reason="InvalidImageName"} 0 - kube_pod_init_container_status_running{container="initcontainer1",namespace="ns1",pod="pod1"} 1 - kube_pod_init_container_status_terminated_reason{container="initcontainer1",namespace="ns1",pod="pod1",reason="Completed"} 0 - kube_pod_init_container_status_terminated_reason{container="initcontainer1",namespace="ns1",pod="pod1",reason="ContainerCannotRun"} 0 - kube_pod_init_container_status_terminated_reason{container="initcontainer1",namespace="ns1",pod="pod1",reason="Error"} 0 - kube_pod_init_container_status_terminated_reason{container="initcontainer1",namespace="ns1",pod="pod1",reason="Evicted"} 0 - kube_pod_init_container_status_terminated_reason{container="initcontainer1",namespace="ns1",pod="pod1",reason="OOMKilled"} 0 - kube_pod_init_container_status_terminated_reason{container="initcontainer1",namespace="ns1",pod="pod1",reason="DeadlineExceeded"} 0 - kube_pod_init_container_status_terminated{container="initcontainer1",namespace="ns1",pod="pod1"} 0 - kube_pod_init_container_status_waiting{container="initcontainer1",namespace="ns1",pod="pod1"} 0 - kube_pod_init_container_status_waiting_reason{container="initcontainer1",namespace="ns1",pod="pod1",reason="ContainerCreating"} 0 - kube_pod_init_container_status_waiting_reason{container="initcontainer1",namespace="ns1",pod="pod1",reason="ImagePullBackOff"} 0 - kube_pod_init_container_status_waiting_reason{container="initcontainer1",namespace="ns1",pod="pod1",reason="CrashLoopBackOff"} 0 - kube_pod_init_container_status_waiting_reason{container="initcontainer1",namespace="ns1",pod="pod1",reason="ErrImagePull"} 0 - kube_pod_init_container_status_waiting_reason{container="initcontainer1",namespace="ns1",pod="pod1",reason="CreateContainerConfigError"} 0 - kube_pod_init_container_status_waiting_reason{container="initcontainer1",namespace="ns1",pod="pod1",reason="CreateContainerError"} 0 - kube_pod_init_container_status_waiting_reason{container="initcontainer1",namespace="ns1",pod="pod1",reason="InvalidImageName"} 0 + kube_pod_container_state_started{container="container1",namespace="ns1",pod="pod1",uid="uid1"} 1.501777018e+09 + kube_pod_container_status_running{container="container1",namespace="ns1",pod="pod1",uid="uid1"} 1 + kube_pod_container_status_terminated{container="container1",namespace="ns1",pod="pod1",uid="uid1"} 0 + kube_pod_container_status_waiting{container="container1",namespace="ns1",pod="pod1",uid="uid1"} 0 + kube_pod_init_container_status_running{container="initcontainer1",namespace="ns1",pod="pod1",uid="uid1"} 1 + kube_pod_init_container_status_terminated{container="initcontainer1",namespace="ns1",pod="pod1",uid="uid1"} 0 + kube_pod_init_container_status_waiting{container="initcontainer1",namespace="ns1",pod="pod1",uid="uid1"} 0 `, MetricNames: []string{ "kube_pod_container_status_running", + "kube_pod_container_state_started", "kube_pod_container_status_waiting", - "kube_pod_container_status_waiting_reason", "kube_pod_container_status_terminated", "kube_pod_container_status_terminated_reason", "kube_pod_init_container_status_running", @@ -372,11 +487,82 @@ func TestPodStore(t *testing.T) { "kube_pod_init_container_status_terminated_reason", }, }, + { + Obj: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns1", + UID: "uid1", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container1", + Image: "k8s.gcr.io/hyperkube1_spec", + }, + }, + }, + Status: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "container1", + State: v1.ContainerState{ + Terminated: &v1.ContainerStateTerminated{ + StartedAt: metav1.Time{ + Time: time.Unix(1501777018, 0), + }, + Reason: "Completed", + }, + }, + }, + }, + }, + }, + Want: ` + # HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state. + # HELP kube_pod_container_state_started [STABLE] Start time in unix timestamp for a pod container. + # HELP kube_pod_container_status_terminated [STABLE] Describes whether the container is currently in terminated state. + # HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state. + # HELP kube_pod_container_status_waiting [STABLE] Describes whether the container is currently in waiting state. + # HELP kube_pod_container_status_waiting_reason [STABLE] Describes the reason the container is currently in waiting state. + # TYPE kube_pod_container_status_running gauge + # TYPE kube_pod_container_state_started gauge + # TYPE kube_pod_container_status_terminated gauge + # TYPE kube_pod_container_status_terminated_reason gauge + # TYPE kube_pod_container_status_waiting gauge + # TYPE kube_pod_container_status_waiting_reason gauge + kube_pod_container_state_started{container="container1",namespace="ns1",pod="pod1",uid="uid1"} 1.501777018e+09 + kube_pod_container_status_running{container="container1",namespace="ns1",pod="pod1",uid="uid1"} 0 + kube_pod_container_status_terminated_reason{container="container1",namespace="ns1",pod="pod1",reason="Completed",uid="uid1"} 1 + kube_pod_container_status_terminated{container="container1",namespace="ns1",pod="pod1",uid="uid1"} 1 + kube_pod_container_status_waiting{container="container1",namespace="ns1",pod="pod1",uid="uid1"} 0 + `, + MetricNames: []string{ + "kube_pod_container_status_running", + "kube_pod_container_state_started", + "kube_pod_container_status_waiting", + "kube_pod_container_status_terminated", + "kube_pod_container_status_terminated_reason", + }, + }, { Obj: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod2", Namespace: "ns2", + UID: "uid2", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container2", + Image: "k8s.gcr.io/hyperkube2_spec", + }, + { + Name: "container3", + Image: "k8s.gcr.io/hyperkube3_spec", + }, + }, }, Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ @@ -384,7 +570,11 @@ func TestPodStore(t *testing.T) { Name: "container2", State: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{ - Reason: "OOMKilled", + StartedAt: metav1.Time{ + Time: time.Unix(1501777018, 0), + }, + Reason: "OOMKilled", + ExitCode: 137, }, }, }, @@ -400,53 +590,32 @@ func TestPodStore(t *testing.T) { }, }, Want: ` - # HELP kube_pod_container_status_running Describes whether the container is currently in running state. - # HELP kube_pod_container_status_terminated Describes whether the container is currently in terminated state. + # HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state. + # HELP kube_pod_container_state_started [STABLE] Start time in unix timestamp for a pod container. + # HELP kube_pod_container_status_terminated [STABLE] Describes whether the container is currently in terminated state. # HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state. - # HELP kube_pod_container_status_waiting Describes whether the container is currently in waiting state. - # HELP kube_pod_container_status_waiting_reason Describes the reason the container is currently in waiting state. + # HELP kube_pod_container_status_waiting [STABLE] Describes whether the container is currently in waiting state. + # HELP kube_pod_container_status_waiting_reason [STABLE] Describes the reason the container is currently in waiting state. # TYPE kube_pod_container_status_running gauge + # TYPE kube_pod_container_state_started gauge # TYPE kube_pod_container_status_terminated gauge # TYPE kube_pod_container_status_terminated_reason gauge # TYPE kube_pod_container_status_waiting gauge # TYPE kube_pod_container_status_waiting_reason gauge - kube_pod_container_status_running{container="container2",namespace="ns2",pod="pod2"} 0 - kube_pod_container_status_running{container="container3",namespace="ns2",pod="pod2"} 0 - kube_pod_container_status_terminated{container="container2",namespace="ns2",pod="pod2"} 1 - kube_pod_container_status_terminated_reason{container="container2",namespace="ns2",pod="pod2",reason="Completed"} 0 - kube_pod_container_status_terminated_reason{container="container2",namespace="ns2",pod="pod2",reason="ContainerCannotRun"} 0 - kube_pod_container_status_terminated_reason{container="container2",namespace="ns2",pod="pod2",reason="Error"} 0 - kube_pod_container_status_terminated_reason{container="container2",namespace="ns2",pod="pod2",reason="Evicted"} 0 - kube_pod_container_status_terminated_reason{container="container2",namespace="ns2",pod="pod2",reason="OOMKilled"} 1 - kube_pod_container_status_terminated_reason{container="container2",namespace="ns2",pod="pod2",reason="DeadlineExceeded"} 0 - kube_pod_container_status_terminated_reason{container="container3",namespace="ns2",pod="pod2",reason="Completed"} 0 - kube_pod_container_status_terminated_reason{container="container3",namespace="ns2",pod="pod2",reason="ContainerCannotRun"} 0 - kube_pod_container_status_terminated_reason{container="container3",namespace="ns2",pod="pod2",reason="Error"} 0 - kube_pod_container_status_terminated_reason{container="container3",namespace="ns2",pod="pod2",reason="Evicted"} 0 - kube_pod_container_status_terminated_reason{container="container3",namespace="ns2",pod="pod2",reason="OOMKilled"} 0 - kube_pod_container_status_terminated_reason{container="container3",namespace="ns2",pod="pod2",reason="DeadlineExceeded"} 0 - kube_pod_container_status_waiting{container="container2",namespace="ns2",pod="pod2"} 0 - kube_pod_container_status_waiting{container="container3",namespace="ns2",pod="pod2"} 1 - kube_pod_container_status_terminated{container="container3",namespace="ns2",pod="pod2"} 0 - kube_pod_container_status_waiting_reason{container="container2",namespace="ns2",pod="pod2",reason="ContainerCreating"} 0 - kube_pod_container_status_waiting_reason{container="container2",namespace="ns2",pod="pod2",reason="ImagePullBackOff"} 0 - kube_pod_container_status_waiting_reason{container="container2",namespace="ns2",pod="pod2",reason="CrashLoopBackOff"} 0 - kube_pod_container_status_waiting_reason{container="container2",namespace="ns2",pod="pod2",reason="ErrImagePull"} 0 - kube_pod_container_status_waiting_reason{container="container2",namespace="ns2",pod="pod2",reason="CreateContainerConfigError"} 0 - kube_pod_container_status_waiting_reason{container="container2",namespace="ns2",pod="pod2",reason="CreateContainerError"} 0 - kube_pod_container_status_waiting_reason{container="container2",namespace="ns2",pod="pod2",reason="InvalidImageName"} 0 - kube_pod_container_status_waiting_reason{container="container3",namespace="ns2",pod="pod2",reason="ContainerCreating"} 1 - kube_pod_container_status_waiting_reason{container="container3",namespace="ns2",pod="pod2",reason="CrashLoopBackOff"} 0 - kube_pod_container_status_waiting_reason{container="container3",namespace="ns2",pod="pod2",reason="ErrImagePull"} 0 - kube_pod_container_status_waiting_reason{container="container3",namespace="ns2",pod="pod2",reason="ImagePullBackOff"} 0 - kube_pod_container_status_waiting_reason{container="container3",namespace="ns2",pod="pod2",reason="CreateContainerConfigError"} 0 - kube_pod_container_status_waiting_reason{container="container3",namespace="ns2",pod="pod2",reason="CreateContainerError"} 0 - kube_pod_container_status_waiting_reason{container="container3",namespace="ns2",pod="pod2",reason="InvalidImageName"} 0 + kube_pod_container_status_running{container="container2",namespace="ns2",pod="pod2",uid="uid2"} 0 + kube_pod_container_status_running{container="container3",namespace="ns2",pod="pod2",uid="uid2"} 0 + kube_pod_container_state_started{container="container2",namespace="ns2",pod="pod2",uid="uid2"} 1.501777018e+09 + kube_pod_container_status_terminated_reason{container="container2",namespace="ns2",pod="pod2",reason="OOMKilled",uid="uid2"} 1 + kube_pod_container_status_terminated{container="container2",namespace="ns2",pod="pod2",uid="uid2"} 1 + kube_pod_container_status_terminated{container="container3",namespace="ns2",pod="pod2",uid="uid2"} 0 + kube_pod_container_status_waiting_reason{container="container3",namespace="ns2",pod="pod2",reason="ContainerCreating",uid="uid2"} 1 + kube_pod_container_status_waiting{container="container2",namespace="ns2",pod="pod2",uid="uid2"} 0 + kube_pod_container_status_waiting{container="container3",namespace="ns2",pod="pod2",uid="uid2"} 1 `, MetricNames: []string{ "kube_pod_container_status_running", + "kube_pod_container_state_started", "kube_pod_container_status_waiting", - "kube_pod_container_status_waiting_reason", "kube_pod_container_status_terminated", "kube_pod_container_status_terminated_reason", }, @@ -456,6 +625,15 @@ func TestPodStore(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "pod3", Namespace: "ns3", + UID: "uid3", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container4", + Image: "k8s.gcr.io/hyperkube4_spec", + }, + }, }, Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ @@ -471,40 +649,24 @@ func TestPodStore(t *testing.T) { }, }, Want: ` + # HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state. # HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state. - # HELP kube_pod_container_status_running Describes whether the container is currently in running state. - # HELP kube_pod_container_status_terminated Describes whether the container is currently in terminated state. + # HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state. + # HELP kube_pod_container_status_terminated [STABLE] Describes whether the container is currently in terminated state. # HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state. - # HELP kube_pod_container_status_waiting Describes whether the container is currently in waiting state. - # HELP kube_pod_container_status_waiting_reason Describes the reason the container is currently in waiting state. + # HELP kube_pod_container_status_waiting [STABLE] Describes whether the container is currently in waiting state. + # HELP kube_pod_container_status_waiting_reason [STABLE] Describes the reason the container is currently in waiting state. + # TYPE kube_pod_container_status_last_terminated_exitcode gauge # TYPE kube_pod_container_status_last_terminated_reason gauge # TYPE kube_pod_container_status_running gauge # TYPE kube_pod_container_status_terminated gauge # TYPE kube_pod_container_status_terminated_reason gauge # TYPE kube_pod_container_status_waiting gauge # TYPE kube_pod_container_status_waiting_reason gauge - kube_pod_container_status_running{container="container4",namespace="ns3",pod="pod3"} 0 - kube_pod_container_status_terminated{container="container4",namespace="ns3",pod="pod3"} 0 -kube_pod_container_status_terminated_reason{container="container4",namespace="ns3",pod="pod3",reason="Completed"} 0 - kube_pod_container_status_terminated_reason{container="container4",namespace="ns3",pod="pod3",reason="ContainerCannotRun"} 0 - kube_pod_container_status_terminated_reason{container="container4",namespace="ns3",pod="pod3",reason="Error"} 0 - kube_pod_container_status_terminated_reason{container="container4",namespace="ns3",pod="pod3",reason="Evicted"} 0 - kube_pod_container_status_terminated_reason{container="container4",namespace="ns3",pod="pod3",reason="OOMKilled"} 0 - kube_pod_container_status_terminated_reason{container="container4",namespace="ns3",pod="pod3",reason="DeadlineExceeded"} 0 - kube_pod_container_status_waiting{container="container4",namespace="ns3",pod="pod3"} 1 -kube_pod_container_status_waiting_reason{container="container4",namespace="ns3",pod="pod3",reason="ContainerCreating"} 0 - kube_pod_container_status_waiting_reason{container="container4",namespace="ns3",pod="pod3",reason="ImagePullBackOff"} 0 - kube_pod_container_status_waiting_reason{container="container4",namespace="ns3",pod="pod3",reason="CrashLoopBackOff"} 1 - kube_pod_container_status_waiting_reason{container="container4",namespace="ns3",pod="pod3",reason="ErrImagePull"} 0 - kube_pod_container_status_waiting_reason{container="container4",namespace="ns3",pod="pod3",reason="CreateContainerConfigError"} 0 - kube_pod_container_status_waiting_reason{container="container4",namespace="ns3",pod="pod3",reason="CreateContainerError"} 0 - kube_pod_container_status_waiting_reason{container="container4",namespace="ns3",pod="pod3",reason="InvalidImageName"} 0 -kube_pod_container_status_last_terminated_reason{container="container4",namespace="ns3",pod="pod3",reason="Completed"} 0 - kube_pod_container_status_last_terminated_reason{container="container4",namespace="ns3",pod="pod3",reason="ContainerCannotRun"} 0 - kube_pod_container_status_last_terminated_reason{container="container4",namespace="ns3",pod="pod3",reason="Error"} 0 - kube_pod_container_status_last_terminated_reason{container="container4",namespace="ns3",pod="pod3",reason="Evicted"} 0 - kube_pod_container_status_last_terminated_reason{container="container4",namespace="ns3",pod="pod3",reason="OOMKilled"} 0 - kube_pod_container_status_last_terminated_reason{container="container4",namespace="ns3",pod="pod3",reason="DeadlineExceeded"} 0 + kube_pod_container_status_running{container="container4",namespace="ns3",pod="pod3",uid="uid3"} 0 + kube_pod_container_status_terminated{container="container4",namespace="ns3",pod="pod3",uid="uid3"} 0 + kube_pod_container_status_waiting_reason{container="container4",namespace="ns3",pod="pod3",reason="CrashLoopBackOff",uid="uid3"} 1 + kube_pod_container_status_waiting{container="container4",namespace="ns3",pod="pod3",uid="uid3"} 1 `, MetricNames: []string{ "kube_pod_container_status_running", @@ -515,14 +677,14 @@ kube_pod_container_status_last_terminated_reason{container="container4",namespac "kube_pod_container_status_terminated_reason", "kube_pod_container_status_waiting", "kube_pod_container_status_waiting_reason", - "kube_pod_container_status_waiting_reason", - "kube_pod_container_status_waiting_reason", - "kube_pod_container_status_waiting_reason", - "kube_pod_container_status_waiting_reason", "kube_pod_container_status_last_terminated_reason", "kube_pod_container_status_last_terminated_reason", "kube_pod_container_status_last_terminated_reason", "kube_pod_container_status_last_terminated_reason", + "kube_pod_container_status_last_terminated_exitcode", + "kube_pod_container_status_last_terminated_exitcode", + "kube_pod_container_status_last_terminated_exitcode", + "kube_pod_container_status_last_terminated_exitcode", }, }, { @@ -531,17 +693,31 @@ kube_pod_container_status_last_terminated_reason{container="container4",namespac ObjectMeta: metav1.ObjectMeta{ Name: "pod6", Namespace: "ns6", + UID: "uid6", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container7", + Image: "k8s.gcr.io/hyperkube7_spec", + }, + }, }, Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ { Name: "container7", State: v1.ContainerState{ - Running: &v1.ContainerStateRunning{}, + Running: &v1.ContainerStateRunning{ + StartedAt: metav1.Time{ + Time: time.Unix(1501777018, 0), + }, + }, }, LastTerminationState: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{ - Reason: "OOMKilled", + Reason: "OOMKilled", + ExitCode: 137, }, }, }, @@ -550,47 +726,36 @@ kube_pod_container_status_last_terminated_reason{container="container4",namespac }, Want: ` # HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state. - # HELP kube_pod_container_status_running Describes whether the container is currently in running state. - # HELP kube_pod_container_status_terminated Describes whether the container is currently in terminated state. + # HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state. + # HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state. + # HELP kube_pod_container_status_terminated [STABLE] Describes whether the container is currently in terminated state. # HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state. - # HELP kube_pod_container_status_waiting Describes whether the container is currently in waiting state. - # HELP kube_pod_container_status_waiting_reason Describes the reason the container is currently in waiting state. + # HELP kube_pod_container_status_waiting [STABLE] Describes whether the container is currently in waiting state. + # HELP kube_pod_container_status_waiting_reason [STABLE] Describes the reason the container is currently in waiting state. + # HELP kube_pod_container_state_started [STABLE] Start time in unix timestamp for a pod container. # TYPE kube_pod_container_status_last_terminated_reason gauge + # TYPE kube_pod_container_status_last_terminated_exitcode gauge # TYPE kube_pod_container_status_running gauge # TYPE kube_pod_container_status_terminated gauge # TYPE kube_pod_container_status_terminated_reason gauge # TYPE kube_pod_container_status_waiting gauge # TYPE kube_pod_container_status_waiting_reason gauge - kube_pod_container_status_running{container="container7",namespace="ns6",pod="pod6"} 1 - kube_pod_container_status_terminated{container="container7",namespace="ns6",pod="pod6"} 0 -kube_pod_container_status_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="Completed"} 0 - kube_pod_container_status_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="ContainerCannotRun"} 0 - kube_pod_container_status_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="Error"} 0 - kube_pod_container_status_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="Evicted"} 0 - kube_pod_container_status_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="OOMKilled"} 0 - kube_pod_container_status_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="DeadlineExceeded"} 0 - kube_pod_container_status_waiting{container="container7",namespace="ns6",pod="pod6"} 0 -kube_pod_container_status_waiting_reason{container="container7",namespace="ns6",pod="pod6",reason="ContainerCreating"} 0 - kube_pod_container_status_waiting_reason{container="container7",namespace="ns6",pod="pod6",reason="ImagePullBackOff"} 0 - kube_pod_container_status_waiting_reason{container="container7",namespace="ns6",pod="pod6",reason="CrashLoopBackOff"} 0 - kube_pod_container_status_waiting_reason{container="container7",namespace="ns6",pod="pod6",reason="ErrImagePull"} 0 - kube_pod_container_status_waiting_reason{container="container7",namespace="ns6",pod="pod6",reason="CreateContainerConfigError"} 0 - kube_pod_container_status_waiting_reason{container="container7",namespace="ns6",pod="pod6",reason="CreateContainerError"} 0 - kube_pod_container_status_waiting_reason{container="container7",namespace="ns6",pod="pod6",reason="InvalidImageName"} 0 -kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="Completed"} 0 - kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="ContainerCannotRun"} 0 - kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="Error"} 0 - kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="Evicted"} 0 - kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="OOMKilled"} 1 - kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="DeadlineExceeded"} 0 + # TYPE kube_pod_container_state_started gauge + kube_pod_container_status_running{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 1 + kube_pod_container_state_started{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 1.501777018e+09 + kube_pod_container_status_terminated{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 0 + kube_pod_container_status_waiting{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 0 + kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="OOMKilled",uid="uid6"} 1 + kube_pod_container_status_last_terminated_exitcode{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 137 `, MetricNames: []string{ "kube_pod_container_status_last_terminated_reason", + "kube_pod_container_status_last_terminated_exitcode", "kube_pod_container_status_running", + "kube_pod_container_state_started", "kube_pod_container_status_terminated", "kube_pod_container_status_terminated_reason", "kube_pod_container_status_waiting", - "kube_pod_container_status_waiting_reason", }, }, { @@ -598,17 +763,31 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac ObjectMeta: metav1.ObjectMeta{ Name: "pod7", Namespace: "ns7", + UID: "uid7", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container7", + Image: "k8s.gcr.io/hyperkube7_spec", + }, + }, }, Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ { Name: "container7", State: v1.ContainerState{ - Running: &v1.ContainerStateRunning{}, + Running: &v1.ContainerStateRunning{ + StartedAt: metav1.Time{ + Time: time.Unix(1501777018, 0), + }, + }, }, LastTerminationState: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{ - Reason: "DeadlineExceeded", + Reason: "DeadlineExceeded", + ExitCode: 143, }, }, }, @@ -616,48 +795,37 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac }, }, Want: ` + # HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state. # HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state. - # HELP kube_pod_container_status_running Describes whether the container is currently in running state. - # HELP kube_pod_container_status_terminated Describes whether the container is currently in terminated state. + # HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state. + # HELP kube_pod_container_state_started [STABLE] Start time in unix timestamp for a pod container. + # HELP kube_pod_container_status_terminated [STABLE] Describes whether the container is currently in terminated state. # HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state. - # HELP kube_pod_container_status_waiting Describes whether the container is currently in waiting state. - # HELP kube_pod_container_status_waiting_reason Describes the reason the container is currently in waiting state. + # HELP kube_pod_container_status_waiting [STABLE] Describes whether the container is currently in waiting state. + # HELP kube_pod_container_status_waiting_reason [STABLE] Describes the reason the container is currently in waiting state. + # TYPE kube_pod_container_status_last_terminated_exitcode gauge # TYPE kube_pod_container_status_last_terminated_reason gauge # TYPE kube_pod_container_status_running gauge + # TYPE kube_pod_container_state_started gauge # TYPE kube_pod_container_status_terminated gauge # TYPE kube_pod_container_status_terminated_reason gauge # TYPE kube_pod_container_status_waiting gauge # TYPE kube_pod_container_status_waiting_reason gauge - kube_pod_container_status_running{container="container7",namespace="ns7",pod="pod7"} 1 - kube_pod_container_status_terminated{container="container7",namespace="ns7",pod="pod7"} 0 -kube_pod_container_status_terminated_reason{container="container7",namespace="ns7",pod="pod7",reason="Completed"} 0 - kube_pod_container_status_terminated_reason{container="container7",namespace="ns7",pod="pod7",reason="ContainerCannotRun"} 0 - kube_pod_container_status_terminated_reason{container="container7",namespace="ns7",pod="pod7",reason="Error"} 0 - kube_pod_container_status_terminated_reason{container="container7",namespace="ns7",pod="pod7",reason="Evicted"} 0 - kube_pod_container_status_terminated_reason{container="container7",namespace="ns7",pod="pod7",reason="OOMKilled"} 0 - kube_pod_container_status_terminated_reason{container="container7",namespace="ns7",pod="pod7",reason="DeadlineExceeded"} 0 - kube_pod_container_status_waiting{container="container7",namespace="ns7",pod="pod7"} 0 -kube_pod_container_status_waiting_reason{container="container7",namespace="ns7",pod="pod7",reason="ContainerCreating"} 0 - kube_pod_container_status_waiting_reason{container="container7",namespace="ns7",pod="pod7",reason="ImagePullBackOff"} 0 - kube_pod_container_status_waiting_reason{container="container7",namespace="ns7",pod="pod7",reason="CrashLoopBackOff"} 0 - kube_pod_container_status_waiting_reason{container="container7",namespace="ns7",pod="pod7",reason="ErrImagePull"} 0 - kube_pod_container_status_waiting_reason{container="container7",namespace="ns7",pod="pod7",reason="CreateContainerConfigError"} 0 - kube_pod_container_status_waiting_reason{container="container7",namespace="ns7",pod="pod7",reason="CreateContainerError"} 0 - kube_pod_container_status_waiting_reason{container="container7",namespace="ns7",pod="pod7",reason="InvalidImageName"} 0 -kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns7",pod="pod7",reason="Completed"} 0 - kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns7",pod="pod7",reason="ContainerCannotRun"} 0 - kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns7",pod="pod7",reason="Error"} 0 - kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns7",pod="pod7",reason="Evicted"} 0 - kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns7",pod="pod7",reason="OOMKilled"} 0 - kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns7",pod="pod7",reason="DeadlineExceeded"} 1 + kube_pod_container_state_started{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 1.501777018e+09 + kube_pod_container_status_last_terminated_exitcode{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 143 + kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns7",pod="pod7",reason="DeadlineExceeded",uid="uid7"} 1 + kube_pod_container_status_running{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 1 + kube_pod_container_status_terminated{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 0 + kube_pod_container_status_waiting{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 0 `, MetricNames: []string{ "kube_pod_container_status_running", + "kube_pod_container_state_started", "kube_pod_container_status_terminated", "kube_pod_container_status_terminated_reason", "kube_pod_container_status_waiting", - "kube_pod_container_status_waiting_reason", "kube_pod_container_status_last_terminated_reason", + "kube_pod_container_status_last_terminated_exitcode", }, }, { @@ -665,6 +833,15 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac ObjectMeta: metav1.ObjectMeta{ Name: "pod4", Namespace: "ns4", + UID: "uid4", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container5", + Image: "k8s.gcr.io/hyperkube5_spec", + }, + }, }, Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ @@ -680,32 +857,20 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac }, }, Want: ` - # HELP kube_pod_container_status_running Describes whether the container is currently in running state. - # HELP kube_pod_container_status_terminated Describes whether the container is currently in terminated state. + # HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state. + # HELP kube_pod_container_status_terminated [STABLE] Describes whether the container is currently in terminated state. # HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state. - # HELP kube_pod_container_status_waiting Describes whether the container is currently in waiting state. - # HELP kube_pod_container_status_waiting_reason Describes the reason the container is currently in waiting state. + # HELP kube_pod_container_status_waiting [STABLE] Describes whether the container is currently in waiting state. + # HELP kube_pod_container_status_waiting_reason [STABLE] Describes the reason the container is currently in waiting state. # TYPE kube_pod_container_status_running gauge # TYPE kube_pod_container_status_terminated gauge # TYPE kube_pod_container_status_terminated_reason gauge # TYPE kube_pod_container_status_waiting gauge # TYPE kube_pod_container_status_waiting_reason gauge - kube_pod_container_status_running{container="container5",namespace="ns4",pod="pod4"} 0 - kube_pod_container_status_terminated{container="container5",namespace="ns4",pod="pod4"} 0 - kube_pod_container_status_terminated_reason{container="container5",namespace="ns4",pod="pod4",reason="Completed"} 0 - kube_pod_container_status_terminated_reason{container="container5",namespace="ns4",pod="pod4",reason="ContainerCannotRun"} 0 - kube_pod_container_status_terminated_reason{container="container5",namespace="ns4",pod="pod4",reason="Error"} 0 - kube_pod_container_status_terminated_reason{container="container5",namespace="ns4",pod="pod4",reason="Evicted"} 0 - kube_pod_container_status_terminated_reason{container="container5",namespace="ns4",pod="pod4",reason="OOMKilled"} 0 - kube_pod_container_status_terminated_reason{container="container5",namespace="ns4",pod="pod4",reason="DeadlineExceeded"} 0 - kube_pod_container_status_waiting{container="container5",namespace="ns4",pod="pod4"} 1 - kube_pod_container_status_waiting_reason{container="container5",namespace="ns4",pod="pod4",reason="ContainerCreating"} 0 - kube_pod_container_status_waiting_reason{container="container5",namespace="ns4",pod="pod4",reason="ImagePullBackOff"} 1 - kube_pod_container_status_waiting_reason{container="container5",namespace="ns4",pod="pod4",reason="CrashLoopBackOff"} 0 - kube_pod_container_status_waiting_reason{container="container5",namespace="ns4",pod="pod4",reason="ErrImagePull"} 0 - kube_pod_container_status_waiting_reason{container="container5",namespace="ns4",pod="pod4",reason="CreateContainerConfigError"} 0 - kube_pod_container_status_waiting_reason{container="container5",namespace="ns4",pod="pod4",reason="CreateContainerError"} 0 - kube_pod_container_status_waiting_reason{container="container5",namespace="ns4",pod="pod4",reason="InvalidImageName"} 0 + kube_pod_container_status_running{container="container5",namespace="ns4",pod="pod4",uid="uid4"} 0 + kube_pod_container_status_terminated{container="container5",namespace="ns4",pod="pod4",uid="uid4"} 0 + kube_pod_container_status_waiting_reason{container="container5",namespace="ns4",pod="pod4",reason="ImagePullBackOff",uid="uid4"} 1 + kube_pod_container_status_waiting{container="container5",namespace="ns4",pod="pod4",uid="uid4"} 1 `, MetricNames: []string{ "kube_pod_container_status_running", @@ -720,6 +885,15 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac ObjectMeta: metav1.ObjectMeta{ Name: "pod5", Namespace: "ns5", + UID: "uid5", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container6", + Image: "k8s.gcr.io/hyperkube1_spec", + }, + }, }, Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ @@ -735,32 +909,20 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac }, }, Want: ` - # HELP kube_pod_container_status_running Describes whether the container is currently in running state. - # HELP kube_pod_container_status_terminated Describes whether the container is currently in terminated state. + # HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state. + # HELP kube_pod_container_status_terminated [STABLE] Describes whether the container is currently in terminated state. # HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state. - # HELP kube_pod_container_status_waiting Describes whether the container is currently in waiting state. - # HELP kube_pod_container_status_waiting_reason Describes the reason the container is currently in waiting state. + # HELP kube_pod_container_status_waiting [STABLE] Describes whether the container is currently in waiting state. + # HELP kube_pod_container_status_waiting_reason [STABLE] Describes the reason the container is currently in waiting state. # TYPE kube_pod_container_status_running gauge # TYPE kube_pod_container_status_terminated gauge # TYPE kube_pod_container_status_terminated_reason gauge # TYPE kube_pod_container_status_waiting gauge # TYPE kube_pod_container_status_waiting_reason gauge - kube_pod_container_status_running{container="container6",namespace="ns5",pod="pod5"} 0 - kube_pod_container_status_terminated{container="container6",namespace="ns5",pod="pod5"} 0 - kube_pod_container_status_terminated_reason{container="container6",namespace="ns5",pod="pod5",reason="Completed"} 0 - kube_pod_container_status_terminated_reason{container="container6",namespace="ns5",pod="pod5",reason="ContainerCannotRun"} 0 - kube_pod_container_status_terminated_reason{container="container6",namespace="ns5",pod="pod5",reason="Error"} 0 - kube_pod_container_status_terminated_reason{container="container6",namespace="ns5",pod="pod5",reason="Evicted"} 0 - kube_pod_container_status_terminated_reason{container="container6",namespace="ns5",pod="pod5",reason="OOMKilled"} 0 - kube_pod_container_status_terminated_reason{container="container6",namespace="ns5",pod="pod5",reason="DeadlineExceeded"} 0 - kube_pod_container_status_waiting{container="container6",namespace="ns5",pod="pod5"} 1 - kube_pod_container_status_waiting_reason{container="container6",namespace="ns5",pod="pod5",reason="ContainerCreating"} 0 - kube_pod_container_status_waiting_reason{container="container6",namespace="ns5",pod="pod5",reason="ImagePullBackOff"} 0 - kube_pod_container_status_waiting_reason{container="container6",namespace="ns5",pod="pod5",reason="CrashLoopBackOff"} 0 - kube_pod_container_status_waiting_reason{container="container6",namespace="ns5",pod="pod5",reason="ErrImagePull"} 1 - kube_pod_container_status_waiting_reason{container="container6",namespace="ns5",pod="pod5",reason="CreateContainerConfigError"} 0 - kube_pod_container_status_waiting_reason{container="container6",namespace="ns5",pod="pod5",reason="CreateContainerError"} 0 - kube_pod_container_status_waiting_reason{container="container6",namespace="ns5",pod="pod5",reason="InvalidImageName"} 0 + kube_pod_container_status_running{container="container6",namespace="ns5",pod="pod5",uid="uid5"} 0 + kube_pod_container_status_terminated{container="container6",namespace="ns5",pod="pod5",uid="uid5"} 0 + kube_pod_container_status_waiting_reason{container="container6",namespace="ns5",pod="pod5",reason="ErrImagePull",uid="uid5"} 1 + kube_pod_container_status_waiting{container="container6",namespace="ns5",pod="pod5",uid="uid5"} 1 `, MetricNames: []string{ "kube_pod_container_status_running", @@ -775,6 +937,15 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac ObjectMeta: metav1.ObjectMeta{ Name: "pod7", Namespace: "ns7", + UID: "uid7", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container8", + Image: "k8s.gcr.io/hyperkube1_spec", + }, + }, }, Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ @@ -790,32 +961,20 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac }, }, Want: ` - # HELP kube_pod_container_status_running Describes whether the container is currently in running state. - # HELP kube_pod_container_status_terminated Describes whether the container is currently in terminated state. + # HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state. + # HELP kube_pod_container_status_terminated [STABLE] Describes whether the container is currently in terminated state. # HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state. - # HELP kube_pod_container_status_waiting Describes whether the container is currently in waiting state. - # HELP kube_pod_container_status_waiting_reason Describes the reason the container is currently in waiting state. + # HELP kube_pod_container_status_waiting [STABLE] Describes whether the container is currently in waiting state. + # HELP kube_pod_container_status_waiting_reason [STABLE] Describes the reason the container is currently in waiting state. # TYPE kube_pod_container_status_running gauge # TYPE kube_pod_container_status_terminated gauge # TYPE kube_pod_container_status_terminated_reason gauge # TYPE kube_pod_container_status_waiting gauge # TYPE kube_pod_container_status_waiting_reason gauge - kube_pod_container_status_running{container="container8",namespace="ns7",pod="pod7"} 0 - kube_pod_container_status_terminated{container="container8",namespace="ns7",pod="pod7"} 0 - kube_pod_container_status_terminated_reason{container="container8",namespace="ns7",pod="pod7",reason="Completed"} 0 - kube_pod_container_status_terminated_reason{container="container8",namespace="ns7",pod="pod7",reason="ContainerCannotRun"} 0 - kube_pod_container_status_terminated_reason{container="container8",namespace="ns7",pod="pod7",reason="Error"} 0 - kube_pod_container_status_terminated_reason{container="container8",namespace="ns7",pod="pod7",reason="Evicted"} 0 - kube_pod_container_status_terminated_reason{container="container8",namespace="ns7",pod="pod7",reason="OOMKilled"} 0 - kube_pod_container_status_terminated_reason{container="container8",namespace="ns7",pod="pod7",reason="DeadlineExceeded"} 0 - kube_pod_container_status_waiting{container="container8",namespace="ns7",pod="pod7"} 1 - kube_pod_container_status_waiting_reason{container="container8",namespace="ns7",pod="pod7",reason="ContainerCreating"} 0 - kube_pod_container_status_waiting_reason{container="container8",namespace="ns7",pod="pod7",reason="ImagePullBackOff"} 0 - kube_pod_container_status_waiting_reason{container="container8",namespace="ns7",pod="pod7",reason="CrashLoopBackOff"} 0 - kube_pod_container_status_waiting_reason{container="container8",namespace="ns7",pod="pod7",reason="ErrImagePull"} 0 - kube_pod_container_status_waiting_reason{container="container8",namespace="ns7",pod="pod7",reason="CreateContainerError"} 0 - kube_pod_container_status_waiting_reason{container="container8",namespace="ns7",pod="pod7",reason="InvalidImageName"} 0 - kube_pod_container_status_waiting_reason{container="container8",namespace="ns7",pod="pod7",reason="CreateContainerConfigError"} 1 + kube_pod_container_status_running{container="container8",namespace="ns7",pod="pod7",uid="uid7"} 0 + kube_pod_container_status_terminated{container="container8",namespace="ns7",pod="pod7",uid="uid7"} 0 + kube_pod_container_status_waiting_reason{container="container8",namespace="ns7",pod="pod7",reason="CreateContainerConfigError",uid="uid7"} 1 + kube_pod_container_status_waiting{container="container8",namespace="ns7",pod="pod7",uid="uid7"} 1 `, MetricNames: []string{ "kube_pod_container_status_running", @@ -837,46 +996,91 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac Spec: v1.PodSpec{ NodeName: "node1", PriorityClassName: "system-node-critical", + HostNetwork: true, + Containers: []v1.Container{ + { + Image: "k8s.gcr.io/hyperkube2_spec", + }, + }, }, Status: v1.PodStatus{ - HostIP: "1.1.1.1", - PodIP: "1.2.3.4", + HostIP: "1.1.1.1", + PodIP: "1.2.3.4", + PodIPs: []v1.PodIP{ + { + IP: "1.2.3.4", + }, + { + IP: "fc00:1234:5678:90ab:cdef:cafe:f00d:d00d", + }, + }, StartTime: &metav1StartTime, }, }, // TODO: Should it be '1501569018' instead? Want: ` - # HELP kube_pod_completion_time Completion time in unix timestamp for a pod. - # HELP kube_pod_created Unix creation timestamp - # HELP kube_pod_info Information about pod. - # HELP kube_pod_owner Information about the Pod's owner. - # HELP kube_pod_start_time Start time in unix timestamp for a pod. + # HELP kube_pod_completion_time [STABLE] Completion time in unix timestamp for a pod. + # HELP kube_pod_created [STABLE] Unix creation timestamp + # HELP kube_pod_info [STABLE] Information about pod. + # HELP kube_pod_ips Pod IP addresses + # HELP kube_pod_owner [STABLE] Information about the Pod's owner. + # HELP kube_pod_start_time [STABLE] Start time in unix timestamp for a pod. # TYPE kube_pod_completion_time gauge # TYPE kube_pod_created gauge # TYPE kube_pod_info gauge + # TYPE kube_pod_ips gauge # TYPE kube_pod_owner gauge # TYPE kube_pod_start_time gauge - kube_pod_created{namespace="ns1",pod="pod1"} 1.5e+09 - kube_pod_info{created_by_kind="",created_by_name="",host_ip="1.1.1.1",namespace="ns1",node="node1",pod="pod1",pod_ip="1.2.3.4",uid="abc-123-xxx",priority_class="system-node-critical"} 1 - kube_pod_start_time{namespace="ns1",pod="pod1"} 1.501569018e+09 - kube_pod_owner{namespace="ns1",owner_is_controller="",owner_kind="",owner_name="",pod="pod1"} 1 + kube_pod_created{namespace="ns1",pod="pod1",uid="abc-123-xxx"} 1.5e+09 + kube_pod_info{created_by_kind="",created_by_name="",host_ip="1.1.1.1",namespace="ns1",node="node1",pod="pod1",pod_ip="1.2.3.4",uid="abc-123-xxx",priority_class="system-node-critical",host_network="true"} 1 + kube_pod_ips{namespace="ns1",pod="pod1",uid="abc-123-xxx",ip="1.2.3.4",ip_family="4"} 1 + kube_pod_ips{namespace="ns1",pod="pod1",uid="abc-123-xxx",ip="fc00:1234:5678:90ab:cdef:cafe:f00d:d00d",ip_family="6"} 1 + kube_pod_start_time{namespace="ns1",pod="pod1",uid="abc-123-xxx"} 1.501569018e+09 + kube_pod_owner{namespace="ns1",owner_is_controller="",owner_kind="",owner_name="",pod="pod1",uid="abc-123-xxx"} 1 `, - MetricNames: []string{"kube_pod_created", "kube_pod_info", "kube_pod_start_time", "kube_pod_completion_time", "kube_pod_owner"}, + MetricNames: []string{"kube_pod_created", "kube_pod_info", "kube_pod_ips", "kube_pod_start_time", "kube_pod_completion_time", "kube_pod_owner"}, + }, + { + Obj: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, + Namespace: "ns1", + UID: "abc-123-xxx", + DeletionTimestamp: &metav1.Time{Time: time.Unix(1800000000, 0)}, + }, + Spec: v1.PodSpec{ + NodeName: "node1", + PriorityClassName: "system-node-critical", + }, + Status: v1.PodStatus{ + HostIP: "1.1.1.1", + PodIP: "1.2.3.4", + StartTime: &metav1StartTime, + }, + }, + Want: ` + # HELP kube_pod_deletion_timestamp Unix deletion timestamp + # TYPE kube_pod_deletion_timestamp gauge + kube_pod_deletion_timestamp{namespace="ns1",pod="pod1",uid="abc-123-xxx"} 1.8e+09 +`, + MetricNames: []string{"kube_pod_deletion_timestamp"}, }, { Obj: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod2", Namespace: "ns2", + UID: "uid2", }, Spec: v1.PodSpec{ RestartPolicy: v1.RestartPolicyAlways, }, }, Want: ` - # HELP kube_pod_restart_policy Describes the restart policy in use by this pod. + # HELP kube_pod_restart_policy [STABLE] Describes the restart policy in use by this pod. # TYPE kube_pod_restart_policy gauge - kube_pod_restart_policy{namespace="ns2",pod="pod2",type="Always"} 1 + kube_pod_restart_policy{namespace="ns2",pod="pod2",type="Always",uid="uid2"} 1 `, MetricNames: []string{"kube_pod_restart_policy"}, }, @@ -885,15 +1089,16 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac ObjectMeta: metav1.ObjectMeta{ Name: "pod2", Namespace: "ns2", + UID: "uid2", }, Spec: v1.PodSpec{ RestartPolicy: v1.RestartPolicyOnFailure, }, }, Want: ` - # HELP kube_pod_restart_policy Describes the restart policy in use by this pod. + # HELP kube_pod_restart_policy [STABLE] Describes the restart policy in use by this pod. # TYPE kube_pod_restart_policy gauge - kube_pod_restart_policy{namespace="ns2",pod="pod2",type="OnFailure"} 1 + kube_pod_restart_policy{namespace="ns2",pod="pod2",type="OnFailure",uid="uid2"} 1 `, MetricNames: []string{"kube_pod_restart_policy"}, }, @@ -913,6 +1118,20 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac }, Spec: v1.PodSpec{ NodeName: "node2", + Containers: []v1.Container{ + { + Name: "container2_1", + Image: "k8s.gcr.io/hyperkube2_spec", + }, + { + Name: "container2_2", + Image: "k8s.gcr.io/hyperkube2_spec", + }, + { + Name: "container2_3", + Image: "k8s.gcr.io/hyperkube2_spec", + }, + }, }, Status: v1.PodStatus{ HostIP: "1.1.1.1", @@ -961,19 +1180,19 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac }, }, Want: ` - # HELP kube_pod_completion_time Completion time in unix timestamp for a pod. - # HELP kube_pod_created Unix creation timestamp - # HELP kube_pod_info Information about pod. - # HELP kube_pod_owner Information about the Pod's owner. - # HELP kube_pod_start_time Start time in unix timestamp for a pod. + # HELP kube_pod_completion_time [STABLE] Completion time in unix timestamp for a pod. + # HELP kube_pod_created [STABLE] Unix creation timestamp + # HELP kube_pod_info [STABLE] Information about pod. + # HELP kube_pod_owner [STABLE] Information about the Pod's owner. + # HELP kube_pod_start_time [STABLE] Start time in unix timestamp for a pod. # TYPE kube_pod_completion_time gauge # TYPE kube_pod_created gauge # TYPE kube_pod_info gauge # TYPE kube_pod_owner gauge # TYPE kube_pod_start_time gauge - kube_pod_info{created_by_kind="ReplicaSet",created_by_name="rs-name",host_ip="1.1.1.1",namespace="ns2",node="node2",pod="pod2",pod_ip="2.3.4.5",uid="abc-456-xxx",priority_class=""} 1 - kube_pod_completion_time{namespace="ns2",pod="pod2"} 1.501888018e+09 - kube_pod_owner{namespace="ns2",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="rs-name",pod="pod2"} 1 + kube_pod_info{created_by_kind="ReplicaSet",created_by_name="rs-name",host_ip="1.1.1.1",namespace="ns2",node="node2",pod="pod2",pod_ip="2.3.4.5",uid="abc-456-xxx",priority_class="",host_network="false"} 1 + kube_pod_completion_time{namespace="ns2",pod="pod2",uid="abc-456-xxx"} 1.501888018e+09 + kube_pod_owner{namespace="ns2",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="rs-name",pod="pod2",uid="abc-456-xxx"} 1 `, MetricNames: []string{"kube_pod_created", "kube_pod_info", "kube_pod_start_time", "kube_pod_completion_time", "kube_pod_owner"}, }, @@ -982,19 +1201,20 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac ObjectMeta: metav1.ObjectMeta{ Name: "pod1", Namespace: "ns1", + UID: "uid1", }, Status: v1.PodStatus{ Phase: v1.PodRunning, }, }, Want: ` - # HELP kube_pod_status_phase The pods current phase. + # HELP kube_pod_status_phase [STABLE] The pods current phase. # TYPE kube_pod_status_phase gauge - kube_pod_status_phase{namespace="ns1",phase="Failed",pod="pod1"} 0 - kube_pod_status_phase{namespace="ns1",phase="Pending",pod="pod1"} 0 - kube_pod_status_phase{namespace="ns1",phase="Running",pod="pod1"} 1 - kube_pod_status_phase{namespace="ns1",phase="Succeeded",pod="pod1"} 0 - kube_pod_status_phase{namespace="ns1",phase="Unknown",pod="pod1"} 0 + kube_pod_status_phase{namespace="ns1",phase="Failed",pod="pod1",uid="uid1"} 0 + kube_pod_status_phase{namespace="ns1",phase="Pending",pod="pod1",uid="uid1"} 0 + kube_pod_status_phase{namespace="ns1",phase="Running",pod="pod1",uid="uid1"} 1 + kube_pod_status_phase{namespace="ns1",phase="Succeeded",pod="pod1",uid="uid1"} 0 + kube_pod_status_phase{namespace="ns1",phase="Unknown",pod="pod1",uid="uid1"} 0 `, MetricNames: []string{"kube_pod_status_phase"}, }, @@ -1003,19 +1223,20 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac ObjectMeta: metav1.ObjectMeta{ Name: "pod2", Namespace: "ns2", + UID: "uid2", }, Status: v1.PodStatus{ Phase: v1.PodPending, }, }, Want: ` - # HELP kube_pod_status_phase The pods current phase. + # HELP kube_pod_status_phase [STABLE] The pods current phase. # TYPE kube_pod_status_phase gauge - kube_pod_status_phase{namespace="ns2",phase="Failed",pod="pod2"} 0 - kube_pod_status_phase{namespace="ns2",phase="Pending",pod="pod2"} 1 - kube_pod_status_phase{namespace="ns2",phase="Running",pod="pod2"} 0 - kube_pod_status_phase{namespace="ns2",phase="Succeeded",pod="pod2"} 0 - kube_pod_status_phase{namespace="ns2",phase="Unknown",pod="pod2"} 0 + kube_pod_status_phase{namespace="ns2",phase="Failed",pod="pod2",uid="uid2"} 0 + kube_pod_status_phase{namespace="ns2",phase="Pending",pod="pod2",uid="uid2"} 1 + kube_pod_status_phase{namespace="ns2",phase="Running",pod="pod2",uid="uid2"} 0 + kube_pod_status_phase{namespace="ns2",phase="Succeeded",pod="pod2",uid="uid2"} 0 + kube_pod_status_phase{namespace="ns2",phase="Unknown",pod="pod2",uid="uid2"} 0 `, MetricNames: []string{"kube_pod_status_phase"}, }, @@ -1025,19 +1246,20 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac ObjectMeta: metav1.ObjectMeta{ Name: "pod3", Namespace: "ns3", + UID: "uid3", }, Status: v1.PodStatus{ Phase: v1.PodUnknown, }, }, Want: ` - # HELP kube_pod_status_phase The pods current phase. + # HELP kube_pod_status_phase [STABLE] The pods current phase. # TYPE kube_pod_status_phase gauge - kube_pod_status_phase{namespace="ns3",phase="Failed",pod="pod3"} 0 - kube_pod_status_phase{namespace="ns3",phase="Pending",pod="pod3"} 0 - kube_pod_status_phase{namespace="ns3",phase="Running",pod="pod3"} 0 - kube_pod_status_phase{namespace="ns3",phase="Succeeded",pod="pod3"} 0 - kube_pod_status_phase{namespace="ns3",phase="Unknown",pod="pod3"} 1 + kube_pod_status_phase{namespace="ns3",phase="Failed",pod="pod3",uid="uid3"} 0 + kube_pod_status_phase{namespace="ns3",phase="Pending",pod="pod3",uid="uid3"} 0 + kube_pod_status_phase{namespace="ns3",phase="Running",pod="pod3",uid="uid3"} 0 + kube_pod_status_phase{namespace="ns3",phase="Succeeded",pod="pod3",uid="uid3"} 0 + kube_pod_status_phase{namespace="ns3",phase="Unknown",pod="pod3",uid="uid3"} 1 `, MetricNames: []string{"kube_pod_status_phase"}, }, @@ -1046,77 +1268,290 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac ObjectMeta: metav1.ObjectMeta{ Name: "pod4", Namespace: "ns4", + UID: "uid4", DeletionTimestamp: &metav1.Time{}, }, Status: v1.PodStatus{ Phase: v1.PodRunning, - Reason: nodeUnreachablePodReason, + Reason: "NodeLost", }, }, Want: ` - # HELP kube_pod_status_phase The pods current phase. + # HELP kube_pod_status_phase [STABLE] The pods current phase. + # HELP kube_pod_status_reason The pod status reasons # TYPE kube_pod_status_phase gauge - kube_pod_status_phase{namespace="ns4",phase="Failed",pod="pod4"} 0 - kube_pod_status_phase{namespace="ns4",phase="Pending",pod="pod4"} 0 - kube_pod_status_phase{namespace="ns4",phase="Running",pod="pod4"} 0 - kube_pod_status_phase{namespace="ns4",phase="Succeeded",pod="pod4"} 0 - kube_pod_status_phase{namespace="ns4",phase="Unknown",pod="pod4"} 1 + # TYPE kube_pod_status_reason gauge + kube_pod_status_phase{namespace="ns4",phase="Failed",pod="pod4",uid="uid4"} 0 + kube_pod_status_phase{namespace="ns4",phase="Pending",pod="pod4",uid="uid4"} 0 + kube_pod_status_phase{namespace="ns4",phase="Running",pod="pod4",uid="uid4"} 1 + kube_pod_status_phase{namespace="ns4",phase="Succeeded",pod="pod4",uid="uid4"} 0 + kube_pod_status_phase{namespace="ns4",phase="Unknown",pod="pod4",uid="uid4"} 0 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="Evicted",uid="uid4"} 0 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="NodeAffinity",uid="uid4"} 0 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="NodeLost",uid="uid4"} 1 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="Shutdown",uid="uid4"} 0 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="UnexpectedAdmissionError",uid="uid4"} 0 `, - MetricNames: []string{"kube_pod_status_phase"}, + MetricNames: []string{"kube_pod_status_phase", "kube_pod_status_reason"}, }, { Obj: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod1", Namespace: "ns1", + UID: "uid1", }, Status: v1.PodStatus{ - Conditions: []v1.PodCondition{ - { + QOSClass: v1.PodQOSBestEffort, + }, + }, + Want: ` + # HELP kube_pod_status_qos_class The pods current qosClass. + # TYPE kube_pod_status_qos_class gauge + kube_pod_status_qos_class{namespace="ns1",qos_class="BestEffort",pod="pod1",uid="uid1"} 1 + kube_pod_status_qos_class{namespace="ns1",qos_class="Burstable",pod="pod1",uid="uid1"} 0 + kube_pod_status_qos_class{namespace="ns1",qos_class="Guaranteed",pod="pod1",uid="uid1"} 0 +`, + MetricNames: []string{"kube_pod_status_qos_class"}, + }, + { + Obj: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod4", + Namespace: "ns4", + UID: "uid4", + DeletionTimestamp: &metav1.Time{}, + }, + Status: v1.PodStatus{ + Phase: v1.PodRunning, + Reason: "Evicted", + }, + }, + Want: ` + # HELP kube_pod_status_reason The pod status reasons + # TYPE kube_pod_status_reason gauge + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="Evicted",uid="uid4"} 1 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="NodeAffinity",uid="uid4"} 0 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="NodeLost",uid="uid4"} 0 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="Shutdown",uid="uid4"} 0 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="UnexpectedAdmissionError",uid="uid4"} 0 +`, + MetricNames: []string{"kube_pod_status_reason"}, + }, + { + Obj: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod4", + Namespace: "ns4", + UID: "uid4", + DeletionTimestamp: &metav1.Time{}, + }, + Status: v1.PodStatus{ + Phase: v1.PodRunning, + Reason: "UnexpectedAdmissionError", + }, + }, + Want: ` + # HELP kube_pod_status_reason The pod status reasons + # TYPE kube_pod_status_reason gauge + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="Evicted",uid="uid4"} 0 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="NodeAffinity",uid="uid4"} 0 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="NodeLost",uid="uid4"} 0 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="Shutdown",uid="uid4"} 0 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="UnexpectedAdmissionError",uid="uid4"} 1 +`, + MetricNames: []string{"kube_pod_status_reason"}, + }, + { + Obj: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod4", + Namespace: "ns4", + UID: "uid4", + DeletionTimestamp: &metav1.Time{}, + }, + Status: v1.PodStatus{ + Phase: v1.PodRunning, + Reason: "NodeAffinity", + }, + }, + Want: ` + # HELP kube_pod_status_reason The pod status reasons + # TYPE kube_pod_status_reason gauge + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="Evicted",uid="uid4"} 0 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="NodeAffinity",uid="uid4"} 1 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="NodeLost",uid="uid4"} 0 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="Shutdown",uid="uid4"} 0 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="UnexpectedAdmissionError",uid="uid4"} 0 +`, + MetricNames: []string{"kube_pod_status_reason"}, + }, + { + Obj: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod4", + Namespace: "ns4", + UID: "uid4", + DeletionTimestamp: &metav1.Time{}, + }, + Status: v1.PodStatus{ + Phase: v1.PodRunning, + Reason: "Shutdown", + }, + }, + Want: ` + # HELP kube_pod_status_reason The pod status reasons + # TYPE kube_pod_status_reason gauge + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="Evicted",uid="uid4"} 0 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="NodeAffinity",uid="uid4"} 0 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="NodeLost",uid="uid4"} 0 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="Shutdown",uid="uid4"} 1 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="UnexpectedAdmissionError",uid="uid4"} 0 +`, + MetricNames: []string{"kube_pod_status_reason"}, + }, + { + Obj: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod4", + Namespace: "ns4", + UID: "uid4", + DeletionTimestamp: &metav1.Time{}, + }, + Status: v1.PodStatus{ + Phase: v1.PodRunning, + Reason: "other reason", + }, + }, + Want: ` + # HELP kube_pod_status_reason The pod status reasons + # TYPE kube_pod_status_reason gauge + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="Evicted",uid="uid4"} 0 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="NodeAffinity",uid="uid4"} 0 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="NodeLost",uid="uid4"} 0 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="Shutdown",uid="uid4"} 0 + kube_pod_status_reason{namespace="ns4",pod="pod4",reason="UnexpectedAdmissionError",uid="uid4"} 0 +`, + MetricNames: []string{"kube_pod_status_reason"}, + }, + { + Obj: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns1", + UID: "uid1", + }, + Status: v1.PodStatus{ + Conditions: []v1.PodCondition{ + { + Type: v1.ContainersReady, + Status: v1.ConditionTrue, + LastTransitionTime: metav1.Time{ + Time: time.Unix(1501666018, 0), + }, + }, + }, + }, + }, + Want: ` + # HELP kube_pod_status_container_ready_time Readiness achieved time in unix timestamp for a pod containers. + # TYPE kube_pod_status_container_ready_time gauge + kube_pod_status_container_ready_time{namespace="ns1",pod="pod1",uid="uid1"} 1.501666018e+09 + `, + MetricNames: []string{"kube_pod_status_container_ready_time"}, + }, + { + Obj: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns1", + UID: "uid1", + }, + Status: v1.PodStatus{ + Conditions: []v1.PodCondition{ + { + Type: v1.ContainersReady, + Status: v1.ConditionFalse, + LastTransitionTime: metav1.Time{ + Time: time.Unix(1501666018, 0), + }, + }, + }, + }, + }, + Want: ` + # HELP kube_pod_status_container_ready_time Readiness achieved time in unix timestamp for a pod containers. + # TYPE kube_pod_status_container_ready_time gauge + `, + MetricNames: []string{"kube_pod_status_container_ready_time"}, + }, + { + Obj: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns1", + UID: "uid1", + }, + Status: v1.PodStatus{ + Conditions: []v1.PodCondition{ + { Type: v1.PodReady, Status: v1.ConditionTrue, + LastTransitionTime: metav1.Time{ + Time: time.Unix(1501666018, 0), + }, }, }, }, }, Want: ` - # HELP kube_pod_status_ready Describes whether the pod is ready to serve requests. + # HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests. + # HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod. # TYPE kube_pod_status_ready gauge - kube_pod_status_ready{condition="false",namespace="ns1",pod="pod1"} 0 - kube_pod_status_ready{condition="true",namespace="ns1",pod="pod1"} 1 - kube_pod_status_ready{condition="unknown",namespace="ns1",pod="pod1"} 0 + # TYPE kube_pod_status_ready_time gauge + kube_pod_status_ready_time{namespace="ns1",pod="pod1",uid="uid1"} 1.501666018e+09 + kube_pod_status_ready{condition="false",namespace="ns1",pod="pod1",uid="uid1"} 0 + kube_pod_status_ready{condition="true",namespace="ns1",pod="pod1",uid="uid1"} 1 + kube_pod_status_ready{condition="unknown",namespace="ns1",pod="pod1",uid="uid1"} 0 `, - MetricNames: []string{"kube_pod_status_ready"}, + MetricNames: []string{"kube_pod_status_ready_time", "kube_pod_status_ready"}, }, { Obj: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod2", Namespace: "ns2", + UID: "uid2", }, Status: v1.PodStatus{ Conditions: []v1.PodCondition{ { Type: v1.PodReady, Status: v1.ConditionFalse, + LastTransitionTime: metav1.Time{ + Time: time.Unix(1501666018, 0), + }, }, }, }, }, Want: ` - # HELP kube_pod_status_ready Describes whether the pod is ready to serve requests. + # HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests. + # HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod. # TYPE kube_pod_status_ready gauge - kube_pod_status_ready{condition="false",namespace="ns2",pod="pod2"} 1 - kube_pod_status_ready{condition="true",namespace="ns2",pod="pod2"} 0 - kube_pod_status_ready{condition="unknown",namespace="ns2",pod="pod2"} 0 + # TYPE kube_pod_status_ready_time gauge + kube_pod_status_ready{condition="false",namespace="ns2",pod="pod2",uid="uid2"} 1 + kube_pod_status_ready{condition="true",namespace="ns2",pod="pod2",uid="uid2"} 0 + kube_pod_status_ready{condition="unknown",namespace="ns2",pod="pod2",uid="uid2"} 0 `, - MetricNames: []string{"kube_pod_status_ready"}, + MetricNames: []string{"kube_pod_status_ready_time", "kube_pod_status_ready"}, }, { Obj: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod1", Namespace: "ns1", + UID: "uid1", }, Status: v1.PodStatus{ Conditions: []v1.PodCondition{ @@ -1131,14 +1566,14 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac }, }, Want: ` - # HELP kube_pod_status_scheduled Describes the status of the scheduling process for the pod. - # HELP kube_pod_status_scheduled_time Unix timestamp when pod moved into scheduled status + # HELP kube_pod_status_scheduled [STABLE] Describes the status of the scheduling process for the pod. + # HELP kube_pod_status_scheduled_time [STABLE] Unix timestamp when pod moved into scheduled status # TYPE kube_pod_status_scheduled gauge # TYPE kube_pod_status_scheduled_time gauge - kube_pod_status_scheduled_time{namespace="ns1",pod="pod1"} 1.501666018e+09 - kube_pod_status_scheduled{condition="false",namespace="ns1",pod="pod1"} 0 - kube_pod_status_scheduled{condition="true",namespace="ns1",pod="pod1"} 1 - kube_pod_status_scheduled{condition="unknown",namespace="ns1",pod="pod1"} 0 + kube_pod_status_scheduled_time{namespace="ns1",pod="pod1",uid="uid1"} 1.501666018e+09 + kube_pod_status_scheduled{condition="false",namespace="ns1",pod="pod1",uid="uid1"} 0 + kube_pod_status_scheduled{condition="true",namespace="ns1",pod="pod1",uid="uid1"} 1 + kube_pod_status_scheduled{condition="unknown",namespace="ns1",pod="pod1",uid="uid1"} 0 `, MetricNames: []string{"kube_pod_status_scheduled", "kube_pod_status_scheduled_time"}, }, @@ -1147,6 +1582,7 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac ObjectMeta: metav1.ObjectMeta{ Name: "pod2", Namespace: "ns2", + UID: "uid2", }, Status: v1.PodStatus{ Conditions: []v1.PodCondition{ @@ -1158,13 +1594,13 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac }, }, Want: ` - # HELP kube_pod_status_scheduled Describes the status of the scheduling process for the pod. - # HELP kube_pod_status_scheduled_time Unix timestamp when pod moved into scheduled status + # HELP kube_pod_status_scheduled [STABLE] Describes the status of the scheduling process for the pod. + # HELP kube_pod_status_scheduled_time [STABLE] Unix timestamp when pod moved into scheduled status # TYPE kube_pod_status_scheduled gauge # TYPE kube_pod_status_scheduled_time gauge - kube_pod_status_scheduled{condition="false",namespace="ns2",pod="pod2"} 1 - kube_pod_status_scheduled{condition="true",namespace="ns2",pod="pod2"} 0 - kube_pod_status_scheduled{condition="unknown",namespace="ns2",pod="pod2"} 0 + kube_pod_status_scheduled{condition="false",namespace="ns2",pod="pod2",uid="uid2"} 1 + kube_pod_status_scheduled{condition="true",namespace="ns2",pod="pod2",uid="uid2"} 0 + kube_pod_status_scheduled{condition="unknown",namespace="ns2",pod="pod2",uid="uid2"} 0 `, MetricNames: []string{"kube_pod_status_scheduled", "kube_pod_status_scheduled_time"}, }, @@ -1173,6 +1609,7 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac ObjectMeta: metav1.ObjectMeta{ Name: "pod2", Namespace: "ns2", + UID: "uid2", }, Status: v1.PodStatus{ Conditions: []v1.PodCondition{ @@ -1186,9 +1623,9 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac }, }, Want: ` - # HELP kube_pod_status_unschedulable Describes the unschedulable status for the pod. + # HELP kube_pod_status_unschedulable [STABLE] Describes the unschedulable status for the pod. # TYPE kube_pod_status_unschedulable gauge - kube_pod_status_unschedulable{namespace="ns2",pod="pod2"} 1 + kube_pod_status_unschedulable{namespace="ns2",pod="pod2",uid="uid2"} 1 `, MetricNames: []string{"kube_pod_status_unschedulable"}, }, @@ -1197,9 +1634,9 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac ObjectMeta: metav1.ObjectMeta{ Name: "pod1", Namespace: "ns1", + UID: "uid1", }, Spec: v1.PodSpec{ - NodeName: "node1", Containers: []v1.Container{ { Name: "pod1_con1", @@ -1258,58 +1695,46 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac }, }, Want: ` - # HELP kube_pod_container_resource_limits The number of requested limit resource by a container. - # HELP kube_pod_container_resource_limits_cpu_cores The limit on cpu cores to be used by a container. - # HELP kube_pod_container_resource_limits_memory_bytes The limit on memory to be used by a container in bytes. - # HELP kube_pod_container_resource_requests The number of requested request resource by a container. - # HELP kube_pod_container_resource_requests_cpu_cores The number of requested cpu cores by a container. - # HELP kube_pod_container_resource_requests_memory_bytes The number of requested memory bytes by a container. - # HELP kube_pod_init_container_resource_limits The number of requested limit resource by the init container. + # HELP kube_pod_container_resource_limits The number of requested limit resource by a container. It is recommended to use the kube_pod_resource_limits metric exposed by kube-scheduler instead, as it is more precise. + # HELP kube_pod_container_resource_requests The number of requested request resource by a container. It is recommended to use the kube_pod_resource_requests metric exposed by kube-scheduler instead, as it is more precise. + # HELP kube_pod_init_container_resource_limits The number of requested limit resource by an init container. + # HELP kube_pod_init_container_resource_requests The number of requested request resource by an init container. # HELP kube_pod_init_container_status_last_terminated_reason Describes the last reason the init container was in terminated state. # TYPE kube_pod_container_resource_limits gauge - # TYPE kube_pod_container_resource_limits_cpu_cores gauge - # TYPE kube_pod_container_resource_limits_memory_bytes gauge # TYPE kube_pod_container_resource_requests gauge - # TYPE kube_pod_container_resource_requests_cpu_cores gauge - # TYPE kube_pod_container_resource_requests_memory_bytes gauge # TYPE kube_pod_init_container_resource_limits gauge + # TYPE kube_pod_init_container_resource_requests gauge # TYPE kube_pod_init_container_status_last_terminated_reason gauge - kube_pod_container_resource_requests_cpu_cores{container="pod1_con1",namespace="ns1",node="node1",pod="pod1"} 0.2 - kube_pod_container_resource_requests_cpu_cores{container="pod1_con2",namespace="ns1",node="node1",pod="pod1"} 0.3 - kube_pod_container_resource_requests_memory_bytes{container="pod1_con1",namespace="ns1",node="node1",pod="pod1"} 1e+08 - kube_pod_container_resource_requests_memory_bytes{container="pod1_con2",namespace="ns1",node="node1",pod="pod1"} 2e+08 - kube_pod_container_resource_limits_cpu_cores{container="pod1_con1",namespace="ns1",node="node1",pod="pod1"} 0.2 - kube_pod_container_resource_limits_cpu_cores{container="pod1_con2",namespace="ns1",node="node1",pod="pod1"} 0.3 - kube_pod_container_resource_limits_memory_bytes{container="pod1_con1",namespace="ns1",node="node1",pod="pod1"} 1e+08 - kube_pod_container_resource_limits_memory_bytes{container="pod1_con2",namespace="ns1",node="node1",pod="pod1"} 2e+08 - kube_pod_container_resource_requests{container="pod1_con1",namespace="ns1",node="node1",pod="pod1",resource="cpu",unit="core"} 0.2 - kube_pod_container_resource_requests{container="pod1_con2",namespace="ns1",node="node1",pod="pod1",resource="cpu",unit="core"} 0.3 - kube_pod_container_resource_requests{container="pod1_con1",namespace="ns1",node="node1",pod="pod1",resource="nvidia_com_gpu",unit="integer"} 1 - kube_pod_container_resource_requests{container="pod1_con1",namespace="ns1",node="node1",pod="pod1",resource="memory",unit="byte"} 1e+08 - kube_pod_container_resource_requests{container="pod1_con2",namespace="ns1",node="node1",pod="pod1",resource="memory",unit="byte"} 2e+08 - kube_pod_container_resource_requests{container="pod1_con1",namespace="ns1",node="node1",pod="pod1",resource="storage",unit="byte"} 4e+08 - kube_pod_container_resource_requests{container="pod1_con1",namespace="ns1",node="node1",pod="pod1",resource="ephemeral_storage",unit="byte"} 3e+08 - kube_pod_container_resource_limits{container="pod1_con1",namespace="ns1",node="node1",pod="pod1",resource="cpu",unit="core"} 0.2 - kube_pod_container_resource_limits{container="pod1_con1",namespace="ns1",node="node1",pod="pod1",resource="nvidia_com_gpu",unit="integer"} 1 - kube_pod_container_resource_limits{container="pod1_con2",namespace="ns1",node="node1",pod="pod1",resource="cpu",unit="core"} 0.3 - kube_pod_container_resource_limits{container="pod1_con1",namespace="ns1",node="node1",pod="pod1",resource="memory",unit="byte"} 1e+08 - kube_pod_container_resource_limits{container="pod1_con2",namespace="ns1",node="node1",pod="pod1",resource="memory",unit="byte"} 2e+08 - kube_pod_container_resource_limits{container="pod1_con1",namespace="ns1",node="node1",pod="pod1",resource="storage",unit="byte"} 4e+08 - kube_pod_container_resource_limits{container="pod1_con1",namespace="ns1",node="node1",pod="pod1",resource="ephemeral_storage",unit="byte"} 3e+08 - kube_pod_init_container_resource_limits{container="pod1_initcon1",namespace="ns1",node="node1",pod="pod1",resource="cpu",unit="core"} 0.2 - kube_pod_init_container_resource_limits{container="pod1_initcon1",namespace="ns1",node="node1",pod="pod1",resource="ephemeral_storage",unit="byte"} 3e+08 - kube_pod_init_container_resource_limits{container="pod1_initcon1",namespace="ns1",node="node1",pod="pod1",resource="memory",unit="byte"} 1e+08 - kube_pod_init_container_resource_limits{container="pod1_initcon1",namespace="ns1",node="node1",pod="pod1",resource="nvidia_com_gpu",unit="integer"} 1 - kube_pod_init_container_resource_limits{container="pod1_initcon1",namespace="ns1",node="node1",pod="pod1",resource="storage",unit="byte"} 4e+08 - `, + kube_pod_container_resource_limits{container="pod1_con1",namespace="ns1",node="",pod="pod1",resource="cpu",unit="core",uid="uid1"} 0.2 + kube_pod_container_resource_limits{container="pod1_con1",namespace="ns1",node="",pod="pod1",resource="ephemeral_storage",unit="byte",uid="uid1"} 3e+08 + kube_pod_container_resource_limits{container="pod1_con1",namespace="ns1",node="",pod="pod1",resource="memory",unit="byte",uid="uid1"} 1e+08 + kube_pod_container_resource_limits{container="pod1_con1",namespace="ns1",node="",pod="pod1",resource="nvidia_com_gpu",unit="integer",uid="uid1"} 1 + kube_pod_container_resource_limits{container="pod1_con1",namespace="ns1",node="",pod="pod1",resource="storage",unit="byte",uid="uid1"} 4e+08 + kube_pod_container_resource_limits{container="pod1_con2",namespace="ns1",node="",pod="pod1",resource="cpu",unit="core",uid="uid1"} 0.3 + kube_pod_container_resource_limits{container="pod1_con2",namespace="ns1",node="",pod="pod1",resource="memory",unit="byte",uid="uid1"} 2e+08 + kube_pod_container_resource_requests{container="pod1_con1",namespace="ns1",node="",pod="pod1",resource="cpu",unit="core",uid="uid1"} 0.2 + kube_pod_container_resource_requests{container="pod1_con1",namespace="ns1",node="",pod="pod1",resource="ephemeral_storage",unit="byte",uid="uid1"} 3e+08 + kube_pod_container_resource_requests{container="pod1_con1",namespace="ns1",node="",pod="pod1",resource="memory",unit="byte",uid="uid1"} 1e+08 + kube_pod_container_resource_requests{container="pod1_con1",namespace="ns1",node="",pod="pod1",resource="nvidia_com_gpu",unit="integer",uid="uid1"} 1 + kube_pod_container_resource_requests{container="pod1_con1",namespace="ns1",node="",pod="pod1",resource="storage",unit="byte",uid="uid1"} 4e+08 + kube_pod_container_resource_requests{container="pod1_con2",namespace="ns1",node="",pod="pod1",resource="cpu",unit="core",uid="uid1"} 0.3 + kube_pod_container_resource_requests{container="pod1_con2",namespace="ns1",node="",pod="pod1",resource="memory",unit="byte",uid="uid1"} 2e+08 + kube_pod_init_container_resource_limits{container="pod1_initcon1",namespace="ns1",node="",pod="pod1",resource="cpu",unit="core",uid="uid1"} 0.2 + kube_pod_init_container_resource_limits{container="pod1_initcon1",namespace="ns1",node="",pod="pod1",resource="ephemeral_storage",unit="byte",uid="uid1"} 3e+08 + kube_pod_init_container_resource_limits{container="pod1_initcon1",namespace="ns1",node="",pod="pod1",resource="memory",unit="byte",uid="uid1"} 1e+08 + kube_pod_init_container_resource_limits{container="pod1_initcon1",namespace="ns1",node="",pod="pod1",resource="nvidia_com_gpu",unit="integer",uid="uid1"} 1 + kube_pod_init_container_resource_limits{container="pod1_initcon1",namespace="ns1",node="",pod="pod1",resource="storage",unit="byte",uid="uid1"} 4e+08 + kube_pod_init_container_resource_requests{container="pod1_initcon1",namespace="ns1",node="",pod="pod1",resource="cpu",unit="core",uid="uid1"} 0.2 + kube_pod_init_container_resource_requests{container="pod1_initcon1",namespace="ns1",node="",pod="pod1",resource="ephemeral_storage",unit="byte",uid="uid1"} 3e+08 + kube_pod_init_container_resource_requests{container="pod1_initcon1",namespace="ns1",node="",pod="pod1",resource="memory",unit="byte",uid="uid1"} 1e+08 + kube_pod_init_container_resource_requests{container="pod1_initcon1",namespace="ns1",node="",pod="pod1",resource="nvidia_com_gpu",unit="integer",uid="uid1"} 1 + kube_pod_init_container_resource_requests{container="pod1_initcon1",namespace="ns1",node="",pod="pod1",resource="storage",unit="byte",uid="uid1"} 4e+08 + `, MetricNames: []string{ - "kube_pod_container_resource_requests_cpu_cores", - "kube_pod_container_resource_requests_memory_bytes", - "kube_pod_container_resource_limits_cpu_cores", - "kube_pod_container_resource_limits_memory_bytes", "kube_pod_container_resource_requests", "kube_pod_container_resource_limits", "kube_pod_init_container_resource_limits", + "kube_pod_init_container_resource_requests", "kube_pod_init_container_status_last_terminated_reason", }, }, @@ -1319,9 +1744,9 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac ObjectMeta: metav1.ObjectMeta{ Name: "pod2", Namespace: "ns2", + UID: "uid2", }, Spec: v1.PodSpec{ - NodeName: "node2", Containers: []v1.Container{ { Name: "pod2_con1", @@ -1372,60 +1797,18 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac }, }, Want: ` - # HELP kube_pod_container_resource_limits The number of requested limit resource by a container. - # HELP kube_pod_container_resource_limits_cpu_cores The limit on cpu cores to be used by a container. - # HELP kube_pod_container_resource_limits_memory_bytes The limit on memory to be used by a container in bytes. - # HELP kube_pod_container_resource_requests The number of requested request resource by a container. - # HELP kube_pod_container_resource_requests_cpu_cores The number of requested cpu cores by a container. - # HELP kube_pod_container_resource_requests_memory_bytes The number of requested memory bytes by a container. - # HELP kube_pod_init_container_resource_limits The number of requested limit resource by the init container. - # TYPE kube_pod_container_resource_limits gauge - # TYPE kube_pod_container_resource_limits_cpu_cores gauge - # TYPE kube_pod_container_resource_limits_memory_bytes gauge - # TYPE kube_pod_container_resource_requests gauge - # TYPE kube_pod_container_resource_requests_cpu_cores gauge - # TYPE kube_pod_container_resource_requests_memory_bytes gauge + # HELP kube_pod_init_container_resource_limits The number of requested limit resource by an init container. + # HELP kube_pod_init_container_resource_requests The number of requested request resource by an init container. # TYPE kube_pod_init_container_resource_limits gauge - kube_pod_container_resource_requests_cpu_cores{container="pod2_con1",namespace="ns2",node="node2",pod="pod2"} 0.4 - kube_pod_container_resource_requests_cpu_cores{container="pod2_con2",namespace="ns2",node="node2",pod="pod2"} 0.5 - kube_pod_container_resource_requests_memory_bytes{container="pod2_con1",namespace="ns2",node="node2",pod="pod2"} 3e+08 - kube_pod_container_resource_requests_memory_bytes{container="pod2_con2",namespace="ns2",node="node2",pod="pod2"} 4e+08 - kube_pod_container_resource_limits_cpu_cores{container="pod2_con1",namespace="ns2",node="node2",pod="pod2"} 0.4 - kube_pod_container_resource_limits_cpu_cores{container="pod2_con2",namespace="ns2",node="node2",pod="pod2"} 0.5 - kube_pod_container_resource_limits_memory_bytes{container="pod2_con1",namespace="ns2",node="node2",pod="pod2"} 3e+08 - kube_pod_container_resource_limits_memory_bytes{container="pod2_con2",namespace="ns2",node="node2",pod="pod2"} 4e+08 - kube_pod_container_resource_requests{container="pod2_con1",namespace="ns2",node="node2",pod="pod2",resource="cpu",unit="core"} 0.4 - kube_pod_container_resource_requests{container="pod2_con2",namespace="ns2",node="node2",pod="pod2",resource="cpu",unit="core"} 0.5 - kube_pod_container_resource_requests{container="pod2_con1",namespace="ns2",node="node2",pod="pod2",resource="memory",unit="byte"} 3e+08 - kube_pod_container_resource_requests{container="pod2_con2",namespace="ns2",node="node2",pod="pod2",resource="memory",unit="byte"} 4e+08 - kube_pod_container_resource_limits{container="pod2_con1",namespace="ns2",node="node2",pod="pod2",resource="cpu",unit="core"} 0.4 - kube_pod_container_resource_limits{container="pod2_con2",namespace="ns2",node="node2",pod="pod2",resource="cpu",unit="core"} 0.5 - kube_pod_container_resource_limits{container="pod2_con1",namespace="ns2",node="node2",pod="pod2",resource="memory",unit="byte"} 3e+08 - kube_pod_container_resource_limits{container="pod2_con2",namespace="ns2",node="node2",pod="pod2",resource="memory",unit="byte"} 4e+08 - kube_pod_init_container_resource_limits{container="pod2_initcon1",namespace="ns2",node="node2",pod="pod2",resource="cpu",unit="core"} 0.4 - kube_pod_init_container_resource_limits{container="pod2_initcon1",namespace="ns2",node="node2",pod="pod2",resource="memory",unit="byte"} 3e+08 - `, + # TYPE kube_pod_init_container_resource_requests gauge + kube_pod_init_container_resource_limits{container="pod2_initcon1",namespace="ns2",node="",pod="pod2",resource="cpu",uid="uid2",unit="core"} 0.4 + kube_pod_init_container_resource_limits{container="pod2_initcon1",namespace="ns2",node="",pod="pod2",resource="memory",uid="uid2",unit="byte"} 3e+08 + kube_pod_init_container_resource_requests{container="pod2_initcon1",namespace="ns2",node="",pod="pod2",resource="cpu",uid="uid2",unit="core"} 0.4 + kube_pod_init_container_resource_requests{container="pod2_initcon1",namespace="ns2",node="",pod="pod2",resource="memory",uid="uid2",unit="byte"} 3e+08 + `, MetricNames: []string{ - "kube_pod_container_resource_requests_cpu_cores", - "kube_pod_container_resource_requests_cpu_cores", - "kube_pod_container_resource_requests_memory_bytes", - "kube_pod_container_resource_requests_memory_bytes", - "kube_pod_container_resource_limits_cpu_cores", - "kube_pod_container_resource_limits_cpu_cores", - "kube_pod_container_resource_limits_memory_bytes", - "kube_pod_container_resource_limits_memory_bytes", - "kube_pod_container_resource_requests", - "kube_pod_container_resource_requests", - "kube_pod_container_resource_requests", - "kube_pod_container_resource_requests", - "kube_pod_container_resource_limits", - "kube_pod_container_resource_limits", - "kube_pod_container_resource_limits", - "kube_pod_container_resource_limits", - "kube_pod_init_container_resource_limits", - "kube_pod_init_container_resource_limits", - "kube_pod_init_container_resource_limits", "kube_pod_init_container_resource_limits", + "kube_pod_init_container_resource_requests", }, }, { @@ -1433,6 +1816,7 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac ObjectMeta: metav1.ObjectMeta{ Name: "pod1", Namespace: "ns1", + UID: "uid1", Labels: map[string]string{ "app": "example", }, @@ -1440,9 +1824,9 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac Spec: v1.PodSpec{}, }, Want: ` - # HELP kube_pod_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_pod_labels [STABLE] Kubernetes labels converted to Prometheus labels. # TYPE kube_pod_labels gauge - kube_pod_labels{label_app="example",namespace="ns1",pod="pod1"} 1 + kube_pod_labels{namespace="ns1",pod="pod1",uid="uid1"} 1 `, MetricNames: []string{ "kube_pod_labels", @@ -1453,6 +1837,7 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac ObjectMeta: metav1.ObjectMeta{ Name: "pod1", Namespace: "ns1", + UID: "uid1", Labels: map[string]string{ "app": "example", }, @@ -1489,14 +1874,14 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac }, }, Want: ` - # HELP kube_pod_spec_volumes_persistentvolumeclaims_info Information about persistentvolumeclaim volumes in a pod. - # HELP kube_pod_spec_volumes_persistentvolumeclaims_readonly Describes whether a persistentvolumeclaim is mounted read only. + # HELP kube_pod_spec_volumes_persistentvolumeclaims_info [STABLE] Information about persistentvolumeclaim volumes in a pod. + # HELP kube_pod_spec_volumes_persistentvolumeclaims_readonly [STABLE] Describes whether a persistentvolumeclaim is mounted read only. # TYPE kube_pod_spec_volumes_persistentvolumeclaims_info gauge # TYPE kube_pod_spec_volumes_persistentvolumeclaims_readonly gauge - kube_pod_spec_volumes_persistentvolumeclaims_info{namespace="ns1",persistentvolumeclaim="claim1",pod="pod1",volume="myvol"} 1 - kube_pod_spec_volumes_persistentvolumeclaims_info{namespace="ns1",persistentvolumeclaim="claim2",pod="pod1",volume="my-readonly-vol"} 1 - kube_pod_spec_volumes_persistentvolumeclaims_readonly{namespace="ns1",persistentvolumeclaim="claim1",pod="pod1",volume="myvol"} 0 - kube_pod_spec_volumes_persistentvolumeclaims_readonly{namespace="ns1",persistentvolumeclaim="claim2",pod="pod1",volume="my-readonly-vol"} 1 + kube_pod_spec_volumes_persistentvolumeclaims_info{namespace="ns1",persistentvolumeclaim="claim1",pod="pod1",volume="myvol",uid="uid1"} 1 + kube_pod_spec_volumes_persistentvolumeclaims_info{namespace="ns1",persistentvolumeclaim="claim2",pod="pod1",volume="my-readonly-vol",uid="uid1"} 1 + kube_pod_spec_volumes_persistentvolumeclaims_readonly{namespace="ns1",persistentvolumeclaim="claim1",pod="pod1",volume="myvol",uid="uid1"} 0 + kube_pod_spec_volumes_persistentvolumeclaims_readonly{namespace="ns1",persistentvolumeclaim="claim2",pod="pod1",volume="my-readonly-vol",uid="uid1"} 1 `, MetricNames: []string{ @@ -1504,11 +1889,188 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac "kube_pod_spec_volumes_persistentvolumeclaims_readonly", }, }, + { + Obj: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns1", + UID: "uid1", + Labels: map[string]string{ + "app": "example", + }, + }, + Spec: v1.PodSpec{ + RuntimeClassName: &runtimeclass, + }, + }, + Want: ` + # HELP kube_pod_runtimeclass_name_info The runtimeclass associated with the pod. + # TYPE kube_pod_runtimeclass_name_info gauge + kube_pod_runtimeclass_name_info{namespace="ns1",pod="pod1",runtimeclass_name="foo",uid="uid1"} 1 + `, + MetricNames: []string{ + "kube_pod_runtimeclass_name_info", + }, + }, + { + Obj: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns1", + UID: "uid1", + Labels: map[string]string{ + "app": "example", + }, + }, + Spec: v1.PodSpec{}, + }, + AllowLabelsList: []string{"wildcard-not-first", options.LabelWildcard}, + Want: ` + # HELP kube_pod_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # TYPE kube_pod_labels gauge + kube_pod_labels{namespace="ns1",pod="pod1",uid="uid1"} 1 + `, + MetricNames: []string{ + "kube_pod_labels", + }, + }, + { + Obj: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns1", + UID: "uid1", + Labels: map[string]string{ + "app": "example", + }, + }, + Spec: v1.PodSpec{}, + }, + AllowLabelsList: []string{options.LabelWildcard}, + Want: ` + # HELP kube_pod_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # TYPE kube_pod_labels gauge + kube_pod_labels{label_app="example",namespace="ns1",pod="pod1",uid="uid1"} 1 + `, + MetricNames: []string{ + "kube_pod_labels", + }, + }, + { + Obj: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns1", + UID: "uid1", + }, + Spec: v1.PodSpec{ + NodeSelector: map[string]string{ + "a": "b", + }, + }, + }, + AllowLabelsList: []string{options.LabelWildcard}, + Want: ` + # HELP kube_pod_nodeselectors Describes the Pod nodeSelectors. + # TYPE kube_pod_nodeselectors gauge + kube_pod_nodeselectors{nodeselector_a="b",namespace="ns1",pod="pod1",uid="uid1"} 1 + `, + MetricNames: []string{ + "kube_pod_nodeselectors", + }, + }, + { + Obj: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "ns1", + UID: "uid6", + }, + Spec: v1.PodSpec{ + NodeSelector: map[string]string{ + "kubernetes.io/os": "linux", + "cloud.google.com/gke-accelerator": "nvidia-tesla-t4", + }, + }, + }, + AllowLabelsList: []string{options.LabelWildcard}, + Want: ` + # HELP kube_pod_nodeselectors Describes the Pod nodeSelectors. + # TYPE kube_pod_nodeselectors gauge + kube_pod_nodeselectors{nodeselector_kubernetes_io_os="linux",nodeselector_cloud_google_com_gke_accelerator="nvidia-tesla-t4",namespace="ns1",pod="pod2",uid="uid6"} 1 + `, + MetricNames: []string{ + "kube_pod_nodeselector", + }, + }, + { + Obj: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns1", + UID: "uid1", + Annotations: map[string]string{ + "app": "example", + }, + }, + Spec: v1.PodSpec{}, + }, + AllowAnnotationsList: []string{options.LabelWildcard}, + Want: ` + # HELP kube_pod_annotations Kubernetes annotations converted to Prometheus labels. + # TYPE kube_pod_annotations gauge + kube_pod_annotations{annotation_app="example",namespace="ns1",pod="pod1",uid="uid1"} 1 + `, + MetricNames: []string{ + "kube_pod_annotations", + }, + }, + { + Obj: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns1", + UID: "uid1", + }, + Spec: v1.PodSpec{ + Tolerations: []v1.Toleration{ + { + Key: "key1", + Operator: v1.TolerationOpEqual, + Value: "value1", + Effect: v1.TaintEffectNoSchedule, + }, + { + Key: "key2", + Operator: v1.TolerationOpExists, + }, + { + Key: "key3", + Operator: v1.TolerationOpEqual, + Value: "value3", + }, + { + // an empty toleration to ensure that an empty toleration does not result in a metric + }, + }, + }, + }, + Want: ` + # HELP kube_pod_tolerations Information about the pod tolerations + # TYPE kube_pod_tolerations gauge + kube_pod_tolerations{namespace="ns1",pod="pod1",uid="uid1",key="key1",operator="Equal",value="value1",effect="NoSchedule"} 1 + kube_pod_tolerations{namespace="ns1",pod="pod1",uid="uid1",key="key2",operator="Exists"} 1 + kube_pod_tolerations{namespace="ns1",pod="pod1",uid="uid1",key="key3",operator="Equal",value="value3"} 1 + `, + MetricNames: []string{ + "kube_pod_tolerations", + }, + }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(podMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(podMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(podMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) + c.Headers = generator.ExtractMetricFamilyHeaders(podMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } @@ -1518,12 +2080,29 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac func BenchmarkPodStore(b *testing.B) { b.ReportAllocs() - f := metric.ComposeMetricGenFuncs(podMetricFamilies) + f := generator.ComposeMetricGenFuncs(podMetricFamilies(nil, nil)) pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod1", Namespace: "ns1", + UID: "uid1", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container1", + Image: "k8s.gcr.io/hyperkube1_spec", + }, + { + Name: "container2", + Image: "k8s.gcr.io/hyperkube1_spec", + }, + { + Name: "container3", + Image: "k8s.gcr.io/hyperkube1_spec", + }, + }, }, Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ @@ -1539,12 +2118,13 @@ func BenchmarkPodStore(b *testing.B) { }, LastTerminationState: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{ - Reason: "OOMKilled", + Reason: "OOMKilled", + ExitCode: 137, }, }, }, { - Name: "container1", + Name: "container2", Image: "k8s.gcr.io/hyperkube1", ImageID: "docker://sha256:aaa", ContainerID: "docker://ab123", @@ -1555,12 +2135,13 @@ func BenchmarkPodStore(b *testing.B) { }, LastTerminationState: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{ - Reason: "OOMKilled", + Reason: "OOMKilled", + ExitCode: 137, }, }, }, { - Name: "container1", + Name: "container3", Image: "k8s.gcr.io/hyperkube1", ImageID: "docker://sha256:aaa", ContainerID: "docker://ab123", @@ -1571,7 +2152,8 @@ func BenchmarkPodStore(b *testing.B) { }, LastTerminationState: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{ - Reason: "OOMKilled", + Reason: "OOMKilled", + ExitCode: 137, }, }, }, @@ -1579,7 +2161,7 @@ func BenchmarkPodStore(b *testing.B) { }, } - expectedFamilies := 39 + expectedFamilies := 50 for n := 0; n < b.N; n++ { families := f(pod) if len(families) != expectedFamilies { diff --git a/internal/store/poddisruptionbudget.go b/internal/store/poddisruptionbudget.go index 2bf23f7dba..0afc0a2223 100644 --- a/internal/store/poddisruptionbudget.go +++ b/internal/store/poddisruptionbudget.go @@ -17,25 +17,75 @@ limitations under the License. package store import ( - "k8s.io/api/policy/v1beta1" + "context" + + policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" - "k8s.io/kube-state-metrics/pkg/metric" + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) var ( descPodDisruptionBudgetLabelsDefaultLabels = []string{"namespace", "poddisruptionbudget"} + descPodDisruptionBudgetAnnotationsName = "kube_poddisruptionbudget_annotations" + descPodDisruptionBudgetAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." + descPodDisruptionBudgetLabelsName = "kube_poddisruptionbudget_labels" + descPodDisruptionBudgetLabelsHelp = "Kubernetes labels converted to Prometheus labels." +) - podDisruptionBudgetMetricFamilies = []metric.FamilyGenerator{ - { - Name: "kube_poddisruptionbudget_created", - Type: metric.Gauge, - Help: "Unix creation timestamp", - GenerateFunc: wrapPodDisruptionBudgetFunc(func(p *v1beta1.PodDisruptionBudget) *metric.Family { +func podDisruptionBudgetMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + descPodDisruptionBudgetAnnotationsName, + descPodDisruptionBudgetAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodDisruptionBudgetFunc(func(p *policyv1.PodDisruptionBudget) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", p.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descPodDisruptionBudgetLabelsName, + descPodDisruptionBudgetLabelsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodDisruptionBudgetFunc(func(p *policyv1.PodDisruptionBudget) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", p.Labels, allowLabelsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: labelKeys, + LabelValues: labelValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_poddisruptionbudget_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodDisruptionBudgetFunc(func(p *policyv1.PodDisruptionBudget) *metric.Family { ms := []*metric.Metric{} if !p.CreationTimestamp.IsZero() { @@ -48,12 +98,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_poddisruptionbudget_status_current_healthy", - Type: metric.Gauge, - Help: "Current number of healthy pods", - GenerateFunc: wrapPodDisruptionBudgetFunc(func(p *v1beta1.PodDisruptionBudget) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_poddisruptionbudget_status_current_healthy", + "Current number of healthy pods", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodDisruptionBudgetFunc(func(p *policyv1.PodDisruptionBudget) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -62,12 +114,14 @@ var ( }, } }), - }, - { - Name: "kube_poddisruptionbudget_status_desired_healthy", - Type: metric.Gauge, - Help: "Minimum desired number of healthy pods", - GenerateFunc: wrapPodDisruptionBudgetFunc(func(p *v1beta1.PodDisruptionBudget) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_poddisruptionbudget_status_desired_healthy", + "Minimum desired number of healthy pods", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodDisruptionBudgetFunc(func(p *policyv1.PodDisruptionBudget) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -76,26 +130,30 @@ var ( }, } }), - }, - { - Name: "kube_poddisruptionbudget_status_pod_disruptions_allowed", - Type: metric.Gauge, - Help: "Number of pod disruptions that are currently allowed", - GenerateFunc: wrapPodDisruptionBudgetFunc(func(p *v1beta1.PodDisruptionBudget) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_poddisruptionbudget_status_pod_disruptions_allowed", + "Number of pod disruptions that are currently allowed", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodDisruptionBudgetFunc(func(p *policyv1.PodDisruptionBudget) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { - Value: float64(p.Status.PodDisruptionsAllowed), + Value: float64(p.Status.DisruptionsAllowed), }, }, } }), - }, - { - Name: "kube_poddisruptionbudget_status_expected_pods", - Type: metric.Gauge, - Help: "Total number of pods counted by this disruption budget", - GenerateFunc: wrapPodDisruptionBudgetFunc(func(p *v1beta1.PodDisruptionBudget) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_poddisruptionbudget_status_expected_pods", + "Total number of pods counted by this disruption budget", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodDisruptionBudgetFunc(func(p *policyv1.PodDisruptionBudget) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -104,12 +162,14 @@ var ( }, } }), - }, - { - Name: "kube_poddisruptionbudget_status_observed_generation", - Type: metric.Gauge, - Help: "Most recent generation observed when updating this PDB status", - GenerateFunc: wrapPodDisruptionBudgetFunc(func(p *v1beta1.PodDisruptionBudget) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_poddisruptionbudget_status_observed_generation", + "Most recent generation observed when updating this PDB status", + metric.Gauge, + basemetrics.STABLE, + "", + wrapPodDisruptionBudgetFunc(func(p *policyv1.PodDisruptionBudget) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -118,32 +178,33 @@ var ( }, } }), - }, + ), } -) +} -func wrapPodDisruptionBudgetFunc(f func(*v1beta1.PodDisruptionBudget) *metric.Family) func(interface{}) *metric.Family { +func wrapPodDisruptionBudgetFunc(f func(*policyv1.PodDisruptionBudget) *metric.Family) func(interface{}) *metric.Family { return func(obj interface{}) *metric.Family { - podDisruptionBudget := obj.(*v1beta1.PodDisruptionBudget) + podDisruptionBudget := obj.(*policyv1.PodDisruptionBudget) metricFamily := f(podDisruptionBudget) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descPodDisruptionBudgetLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{podDisruptionBudget.Namespace, podDisruptionBudget.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descPodDisruptionBudgetLabelsDefaultLabels, []string{podDisruptionBudget.Namespace, podDisruptionBudget.Name}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createPodDisruptionBudgetListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createPodDisruptionBudgetListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.PolicyV1beta1().PodDisruptionBudgets(ns).List(opts) + opts.FieldSelector = fieldSelector + return kubeClient.PolicyV1().PodDisruptionBudgets(ns).List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.PolicyV1beta1().PodDisruptionBudgets(ns).Watch(opts) + opts.FieldSelector = fieldSelector + return kubeClient.PolicyV1().PodDisruptionBudgets(ns).Watch(context.TODO(), opts) }, } } diff --git a/internal/store/poddisruptionbudget_test.go b/internal/store/poddisruptionbudget_test.go index 26d175eb68..33586c1ca4 100644 --- a/internal/store/poddisruptionbudget_test.go +++ b/internal/store/poddisruptionbudget_test.go @@ -20,47 +20,55 @@ import ( "testing" "time" - "k8s.io/api/policy/v1beta1" + policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) func TestPodDisruptionBudgetStore(t *testing.T) { // Fixed metadata on type and help text. We prepend this to every expected // output so we only have to modify a single place when doing adjustments. - const metadata = ` - # HELP kube_poddisruptionbudget_created Unix creation timestamp + const labelsAndAnnotationsMetaData = ` + # HELP kube_poddisruptionbudget_annotations Kubernetes annotations converted to Prometheus labels. + # TYPE kube_poddisruptionbudget_annotations gauge + # HELP kube_poddisruptionbudget_labels Kubernetes labels converted to Prometheus labels. + # TYPE kube_poddisruptionbudget_labels gauge + ` + const metadata = labelsAndAnnotationsMetaData + ` + # HELP kube_poddisruptionbudget_created [STABLE] Unix creation timestamp # TYPE kube_poddisruptionbudget_created gauge - # HELP kube_poddisruptionbudget_status_current_healthy Current number of healthy pods + # HELP kube_poddisruptionbudget_status_current_healthy [STABLE] Current number of healthy pods # TYPE kube_poddisruptionbudget_status_current_healthy gauge - # HELP kube_poddisruptionbudget_status_desired_healthy Minimum desired number of healthy pods + # HELP kube_poddisruptionbudget_status_desired_healthy [STABLE] Minimum desired number of healthy pods # TYPE kube_poddisruptionbudget_status_desired_healthy gauge - # HELP kube_poddisruptionbudget_status_pod_disruptions_allowed Number of pod disruptions that are currently allowed + # HELP kube_poddisruptionbudget_status_pod_disruptions_allowed [STABLE] Number of pod disruptions that are currently allowed # TYPE kube_poddisruptionbudget_status_pod_disruptions_allowed gauge - # HELP kube_poddisruptionbudget_status_expected_pods Total number of pods counted by this disruption budget + # HELP kube_poddisruptionbudget_status_expected_pods [STABLE] Total number of pods counted by this disruption budget # TYPE kube_poddisruptionbudget_status_expected_pods gauge - # HELP kube_poddisruptionbudget_status_observed_generation Most recent generation observed when updating this PDB status + # HELP kube_poddisruptionbudget_status_observed_generation [STABLE] Most recent generation observed when updating this PDB status # TYPE kube_poddisruptionbudget_status_observed_generation gauge ` cases := []generateMetricsTestCase{ { - Obj: &v1beta1.PodDisruptionBudget{ + Obj: &policyv1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ Name: "pdb1", CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, Namespace: "ns1", Generation: 21, }, - Status: v1beta1.PodDisruptionBudgetStatus{ - CurrentHealthy: 12, - DesiredHealthy: 10, - PodDisruptionsAllowed: 2, - ExpectedPods: 15, - ObservedGeneration: 111, + Status: policyv1.PodDisruptionBudgetStatus{ + CurrentHealthy: 12, + DesiredHealthy: 10, + DisruptionsAllowed: 2, + ExpectedPods: 15, + ObservedGeneration: 111, }, }, Want: metadata + ` + kube_poddisruptionbudget_annotations{namespace="ns1",poddisruptionbudget="pdb1"} 1 + kube_poddisruptionbudget_labels{namespace="ns1",poddisruptionbudget="pdb1"} 1 kube_poddisruptionbudget_created{namespace="ns1",poddisruptionbudget="pdb1"} 1.5e+09 kube_poddisruptionbudget_status_current_healthy{namespace="ns1",poddisruptionbudget="pdb1"} 12 kube_poddisruptionbudget_status_desired_healthy{namespace="ns1",poddisruptionbudget="pdb1"} 10 @@ -70,21 +78,23 @@ func TestPodDisruptionBudgetStore(t *testing.T) { `, }, { - Obj: &v1beta1.PodDisruptionBudget{ + Obj: &policyv1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ Name: "pdb2", Namespace: "ns2", Generation: 14, }, - Status: v1beta1.PodDisruptionBudgetStatus{ - CurrentHealthy: 8, - DesiredHealthy: 9, - PodDisruptionsAllowed: 0, - ExpectedPods: 10, - ObservedGeneration: 1111, + Status: policyv1.PodDisruptionBudgetStatus{ + CurrentHealthy: 8, + DesiredHealthy: 9, + DisruptionsAllowed: 0, + ExpectedPods: 10, + ObservedGeneration: 1111, }, }, Want: metadata + ` + kube_poddisruptionbudget_annotations{namespace="ns2",poddisruptionbudget="pdb2"} 1 + kube_poddisruptionbudget_labels{namespace="ns2",poddisruptionbudget="pdb2"} 1 kube_poddisruptionbudget_status_current_healthy{namespace="ns2",poddisruptionbudget="pdb2"} 8 kube_poddisruptionbudget_status_desired_healthy{namespace="ns2",poddisruptionbudget="pdb2"} 9 kube_poddisruptionbudget_status_pod_disruptions_allowed{namespace="ns2",poddisruptionbudget="pdb2"} 0 @@ -92,10 +102,40 @@ func TestPodDisruptionBudgetStore(t *testing.T) { kube_poddisruptionbudget_status_observed_generation{namespace="ns2",poddisruptionbudget="pdb2"} 1111 `, }, + { + AllowAnnotationsList: []string{ + "app.k8s.io/owner", + }, + AllowLabelsList: []string{ + "app", + }, + Obj: &policyv1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pdb_with_allowed_labels_and_annotations", + Namespace: "ns", + Annotations: map[string]string{ + "app.k8s.io/owner": "mysql-server", + "foo": "bar", + }, + Labels: map[string]string{ + "app": "mysql-server", + "hello": "world", + }, + }, + }, + Want: labelsAndAnnotationsMetaData + ` + kube_poddisruptionbudget_annotations{annotation_app_k8s_io_owner="mysql-server",namespace="ns",poddisruptionbudget="pdb_with_allowed_labels_and_annotations"} 1 + kube_poddisruptionbudget_labels{label_app="mysql-server",namespace="ns",poddisruptionbudget="pdb_with_allowed_labels_and_annotations"} 1 + `, + MetricNames: []string{ + "kube_poddisruptionbudget_annotations", + "kube_poddisruptionbudget_labels", + }, + }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(podDisruptionBudgetMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(podDisruptionBudgetMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(podDisruptionBudgetMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) + c.Headers = generator.ExtractMetricFamilyHeaders(podDisruptionBudgetMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/replicaset.go b/internal/store/replicaset.go index 7bd070af43..e1c957bf5d 100644 --- a/internal/store/replicaset.go +++ b/internal/store/replicaset.go @@ -17,9 +17,13 @@ limitations under the License. package store import ( + "context" "strconv" - "k8s.io/kube-state-metrics/pkg/metric" + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" v1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -31,15 +35,21 @@ import ( var ( descReplicaSetLabelsDefaultLabels = []string{"namespace", "replicaset"} + descReplicaSetAnnotationsName = "kube_replicaset_annotations" + descReplicaSetAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." descReplicaSetLabelsName = "kube_replicaset_labels" descReplicaSetLabelsHelp = "Kubernetes labels converted to Prometheus labels." +) - replicaSetMetricFamilies = []metric.FamilyGenerator{ - { - Name: "kube_replicaset_created", - Type: metric.Gauge, - Help: "Unix creation timestamp", - GenerateFunc: wrapReplicaSetFunc(func(r *v1.ReplicaSet) *metric.Family { +func replicaSetMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_replicaset_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.STABLE, + "", + wrapReplicaSetFunc(func(r *v1.ReplicaSet) *metric.Family { ms := []*metric.Metric{} if !r.CreationTimestamp.IsZero() { @@ -53,12 +63,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_replicaset_status_replicas", - Type: metric.Gauge, - Help: "The number of replicas per ReplicaSet.", - GenerateFunc: wrapReplicaSetFunc(func(r *v1.ReplicaSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_replicaset_status_replicas", + "The number of replicas per ReplicaSet.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapReplicaSetFunc(func(r *v1.ReplicaSet) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -67,12 +79,14 @@ var ( }, } }), - }, - { - Name: "kube_replicaset_status_fully_labeled_replicas", - Type: metric.Gauge, - Help: "The number of fully labeled replicas per ReplicaSet.", - GenerateFunc: wrapReplicaSetFunc(func(r *v1.ReplicaSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_replicaset_status_fully_labeled_replicas", + "The number of fully labeled replicas per ReplicaSet.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapReplicaSetFunc(func(r *v1.ReplicaSet) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -81,12 +95,14 @@ var ( }, } }), - }, - { - Name: "kube_replicaset_status_ready_replicas", - Type: metric.Gauge, - Help: "The number of ready replicas per ReplicaSet.", - GenerateFunc: wrapReplicaSetFunc(func(r *v1.ReplicaSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_replicaset_status_ready_replicas", + "The number of ready replicas per ReplicaSet.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapReplicaSetFunc(func(r *v1.ReplicaSet) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -95,12 +111,14 @@ var ( }, } }), - }, - { - Name: "kube_replicaset_status_observed_generation", - Type: metric.Gauge, - Help: "The generation observed by the ReplicaSet controller.", - GenerateFunc: wrapReplicaSetFunc(func(r *v1.ReplicaSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_replicaset_status_observed_generation", + "The generation observed by the ReplicaSet controller.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapReplicaSetFunc(func(r *v1.ReplicaSet) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -109,12 +127,14 @@ var ( }, } }), - }, - { - Name: "kube_replicaset_spec_replicas", - Type: metric.Gauge, - Help: "Number of desired pods for a ReplicaSet.", - GenerateFunc: wrapReplicaSetFunc(func(r *v1.ReplicaSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_replicaset_spec_replicas", + "Number of desired pods for a ReplicaSet.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapReplicaSetFunc(func(r *v1.ReplicaSet) *metric.Family { ms := []*metric.Metric{} if r.Spec.Replicas != nil { @@ -127,12 +147,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_replicaset_metadata_generation", - Type: metric.Gauge, - Help: "Sequence number representing a specific generation of the desired state.", - GenerateFunc: wrapReplicaSetFunc(func(r *v1.ReplicaSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_replicaset_metadata_generation", + "Sequence number representing a specific generation of the desired state.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapReplicaSetFunc(func(r *v1.ReplicaSet) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -141,12 +163,14 @@ var ( }, } }), - }, - { - Name: "kube_replicaset_owner", - Type: metric.Gauge, - Help: "Information about the ReplicaSet's owner.", - GenerateFunc: wrapReplicaSetFunc(func(r *v1.ReplicaSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_replicaset_owner", + "Information about the ReplicaSet's owner.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapReplicaSetFunc(func(r *v1.ReplicaSet) *metric.Family { owners := r.GetOwnerReferences() if len(owners) == 0 { @@ -154,7 +178,7 @@ var ( Metrics: []*metric.Metric{ { LabelKeys: []string{"owner_kind", "owner_name", "owner_is_controller"}, - LabelValues: []string{"", "", ""}, + LabelValues: []string{"", "", ""}, Value: 1, }, }, @@ -184,13 +208,34 @@ var ( Metrics: ms, } }), - }, - { - Name: descReplicaSetLabelsName, - Type: metric.Gauge, - Help: descReplicaSetLabelsHelp, - GenerateFunc: wrapReplicaSetFunc(func(d *v1.ReplicaSet) *metric.Family { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(d.Labels) + ), + *generator.NewFamilyGeneratorWithStability( + descReplicaSetAnnotationsName, + descReplicaSetAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapReplicaSetFunc(func(r *v1.ReplicaSet) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", r.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descReplicaSetLabelsName, + descReplicaSetLabelsHelp, + metric.Gauge, + basemetrics.STABLE, + "", + wrapReplicaSetFunc(func(r *v1.ReplicaSet) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", r.Labels, allowLabelsList) return &metric.Family{ Metrics: []*metric.Metric{ { @@ -201,9 +246,9 @@ var ( }, } }), - }, + ), } -) +} func wrapReplicaSetFunc(f func(*v1.ReplicaSet) *metric.Family) func(interface{}) *metric.Family { return func(obj interface{}) *metric.Family { @@ -212,21 +257,22 @@ func wrapReplicaSetFunc(f func(*v1.ReplicaSet) *metric.Family) func(interface{}) metricFamily := f(replicaSet) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descReplicaSetLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{replicaSet.Namespace, replicaSet.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descReplicaSetLabelsDefaultLabels, []string{replicaSet.Namespace, replicaSet.Name}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createReplicaSetListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createReplicaSetListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.AppsV1().ReplicaSets(ns).List(opts) + opts.FieldSelector = fieldSelector + return kubeClient.AppsV1().ReplicaSets(ns).List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.AppsV1().ReplicaSets(ns).Watch(opts) + opts.FieldSelector = fieldSelector + return kubeClient.AppsV1().ReplicaSets(ns).Watch(context.TODO(), opts) }, } } diff --git a/internal/store/replicaset_test.go b/internal/store/replicaset_test.go index 9b919de058..80d7fabd7f 100644 --- a/internal/store/replicaset_test.go +++ b/internal/store/replicaset_test.go @@ -23,7 +23,7 @@ import ( v1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) var ( @@ -36,23 +36,25 @@ func TestReplicaSetStore(t *testing.T) { // Fixed metadata on type and help text. We prepend this to every expected // output so we only have to modify a single place when doing adjustments. const metadata = ` - # HELP kube_replicaset_created Unix creation timestamp + # HELP kube_replicaset_annotations Kubernetes annotations converted to Prometheus labels. + # TYPE kube_replicaset_annotations gauge + # HELP kube_replicaset_created [STABLE] Unix creation timestamp # TYPE kube_replicaset_created gauge - # HELP kube_replicaset_metadata_generation Sequence number representing a specific generation of the desired state. + # HELP kube_replicaset_metadata_generation [STABLE] Sequence number representing a specific generation of the desired state. # TYPE kube_replicaset_metadata_generation gauge - # HELP kube_replicaset_status_replicas The number of replicas per ReplicaSet. + # HELP kube_replicaset_status_replicas [STABLE] The number of replicas per ReplicaSet. # TYPE kube_replicaset_status_replicas gauge - # HELP kube_replicaset_status_fully_labeled_replicas The number of fully labeled replicas per ReplicaSet. + # HELP kube_replicaset_status_fully_labeled_replicas [STABLE] The number of fully labeled replicas per ReplicaSet. # TYPE kube_replicaset_status_fully_labeled_replicas gauge - # HELP kube_replicaset_status_ready_replicas The number of ready replicas per ReplicaSet. + # HELP kube_replicaset_status_ready_replicas [STABLE] The number of ready replicas per ReplicaSet. # TYPE kube_replicaset_status_ready_replicas gauge - # HELP kube_replicaset_status_observed_generation The generation observed by the ReplicaSet controller. + # HELP kube_replicaset_status_observed_generation [STABLE] The generation observed by the ReplicaSet controller. # TYPE kube_replicaset_status_observed_generation gauge - # HELP kube_replicaset_spec_replicas Number of desired pods for a ReplicaSet. + # HELP kube_replicaset_spec_replicas [STABLE] Number of desired pods for a ReplicaSet. # TYPE kube_replicaset_spec_replicas gauge - # HELP kube_replicaset_owner Information about the ReplicaSet's owner. + # HELP kube_replicaset_owner [STABLE] Information about the ReplicaSet's owner. # TYPE kube_replicaset_owner gauge - # HELP kube_replicaset_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_replicaset_labels [STABLE] Kubernetes labels converted to Prometheus labels. # TYPE kube_replicaset_labels gauge ` cases := []generateMetricsTestCase{ @@ -85,7 +87,8 @@ func TestReplicaSetStore(t *testing.T) { }, }, Want: metadata + ` - kube_replicaset_labels{replicaset="rs1",namespace="ns1",label_app="example1"} 1 + kube_replicaset_annotations{replicaset="rs1",namespace="ns1"} 1 + kube_replicaset_labels{replicaset="rs1",namespace="ns1"} 1 kube_replicaset_created{namespace="ns1",replicaset="rs1"} 1.5e+09 kube_replicaset_metadata_generation{namespace="ns1",replicaset="rs1"} 21 kube_replicaset_status_replicas{namespace="ns1",replicaset="rs1"} 5 @@ -118,20 +121,21 @@ func TestReplicaSetStore(t *testing.T) { }, }, Want: metadata + ` - kube_replicaset_labels{replicaset="rs2",namespace="ns2",label_app="example2",label_env="ex"} 1 + kube_replicaset_annotations{replicaset="rs2",namespace="ns2"} 1 + kube_replicaset_labels{replicaset="rs2",namespace="ns2"} 1 kube_replicaset_metadata_generation{namespace="ns2",replicaset="rs2"} 14 kube_replicaset_status_replicas{namespace="ns2",replicaset="rs2"} 0 kube_replicaset_status_observed_generation{namespace="ns2",replicaset="rs2"} 5 kube_replicaset_status_fully_labeled_replicas{namespace="ns2",replicaset="rs2"} 5 kube_replicaset_status_ready_replicas{namespace="ns2",replicaset="rs2"} 0 kube_replicaset_spec_replicas{namespace="ns2",replicaset="rs2"} 0 - kube_replicaset_owner{namespace="ns2",owner_is_controller="",owner_kind="",owner_name="",replicaset="rs2"} 1 + kube_replicaset_owner{namespace="ns2",owner_is_controller="",owner_kind="",owner_name="",replicaset="rs2"} 1 `, }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(replicaSetMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(replicaSetMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(replicaSetMetricFamilies(nil, nil)) + c.Headers = generator.ExtractMetricFamilyHeaders(replicaSetMetricFamilies(nil, nil)) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/replicationcontroller.go b/internal/store/replicationcontroller.go index f3ab3bea59..fee30188f5 100644 --- a/internal/store/replicationcontroller.go +++ b/internal/store/replicationcontroller.go @@ -17,25 +17,32 @@ limitations under the License. package store import ( + "context" + "strconv" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" - "k8s.io/kube-state-metrics/pkg/metric" + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) var ( descReplicationControllerLabelsDefaultLabels = []string{"namespace", "replicationcontroller"} - replicationControllerMetricFamilies = []metric.FamilyGenerator{ - { - Name: "kube_replicationcontroller_created", - Type: metric.Gauge, - Help: "Unix creation timestamp", - GenerateFunc: wrapReplicationControllerFunc(func(r *v1.ReplicationController) *metric.Family { + replicationControllerMetricFamilies = []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_replicationcontroller_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.STABLE, + "", + wrapReplicationControllerFunc(func(r *v1.ReplicationController) *metric.Family { ms := []*metric.Metric{} if !r.CreationTimestamp.IsZero() { @@ -48,12 +55,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_replicationcontroller_status_replicas", - Type: metric.Gauge, - Help: "The number of replicas per ReplicationController.", - GenerateFunc: wrapReplicationControllerFunc(func(r *v1.ReplicationController) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_replicationcontroller_status_replicas", + "The number of replicas per ReplicationController.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapReplicationControllerFunc(func(r *v1.ReplicationController) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -62,12 +71,14 @@ var ( }, } }), - }, - { - Name: "kube_replicationcontroller_status_fully_labeled_replicas", - Type: metric.Gauge, - Help: "The number of fully labeled replicas per ReplicationController.", - GenerateFunc: wrapReplicationControllerFunc(func(r *v1.ReplicationController) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_replicationcontroller_status_fully_labeled_replicas", + "The number of fully labeled replicas per ReplicationController.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapReplicationControllerFunc(func(r *v1.ReplicationController) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -76,12 +87,14 @@ var ( }, } }), - }, - { - Name: "kube_replicationcontroller_status_ready_replicas", - Type: metric.Gauge, - Help: "The number of ready replicas per ReplicationController.", - GenerateFunc: wrapReplicationControllerFunc(func(r *v1.ReplicationController) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_replicationcontroller_status_ready_replicas", + "The number of ready replicas per ReplicationController.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapReplicationControllerFunc(func(r *v1.ReplicationController) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -90,12 +103,14 @@ var ( }, } }), - }, - { - Name: "kube_replicationcontroller_status_available_replicas", - Type: metric.Gauge, - Help: "The number of available replicas per ReplicationController.", - GenerateFunc: wrapReplicationControllerFunc(func(r *v1.ReplicationController) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_replicationcontroller_status_available_replicas", + "The number of available replicas per ReplicationController.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapReplicationControllerFunc(func(r *v1.ReplicationController) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -104,12 +119,14 @@ var ( }, } }), - }, - { - Name: "kube_replicationcontroller_status_observed_generation", - Type: metric.Gauge, - Help: "The generation observed by the ReplicationController controller.", - GenerateFunc: wrapReplicationControllerFunc(func(r *v1.ReplicationController) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_replicationcontroller_status_observed_generation", + "The generation observed by the ReplicationController controller.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapReplicationControllerFunc(func(r *v1.ReplicationController) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -118,12 +135,14 @@ var ( }, } }), - }, - { - Name: "kube_replicationcontroller_spec_replicas", - Type: metric.Gauge, - Help: "Number of desired pods for a ReplicationController.", - GenerateFunc: wrapReplicationControllerFunc(func(r *v1.ReplicationController) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_replicationcontroller_spec_replicas", + "Number of desired pods for a ReplicationController.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapReplicationControllerFunc(func(r *v1.ReplicationController) *metric.Family { ms := []*metric.Metric{} if r.Spec.Replicas != nil { @@ -136,12 +155,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_replicationcontroller_metadata_generation", - Type: metric.Gauge, - Help: "Sequence number representing a specific generation of the desired state.", - GenerateFunc: wrapReplicationControllerFunc(func(r *v1.ReplicationController) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_replicationcontroller_metadata_generation", + "Sequence number representing a specific generation of the desired state.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapReplicationControllerFunc(func(r *v1.ReplicationController) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -150,7 +171,44 @@ var ( }, } }), - }, + ), + *generator.NewFamilyGeneratorWithStability( + "kube_replicationcontroller_owner", + "Information about the ReplicationController's owner.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapReplicationControllerFunc(func(r *v1.ReplicationController) *metric.Family { + labelKeys := []string{"owner_kind", "owner_name", "owner_is_controller"} + ms := []*metric.Metric{} + + owners := r.GetOwnerReferences() + if len(owners) == 0 { + ms = append(ms, &metric.Metric{ + LabelKeys: labelKeys, + LabelValues: []string{"", "", ""}, + Value: 1, + }) + } else { + for _, owner := range owners { + ownerIsController := "false" + if owner.Controller != nil { + ownerIsController = strconv.FormatBool(*owner.Controller) + } + + ms = append(ms, &metric.Metric{ + LabelKeys: labelKeys, + LabelValues: []string{owner.Kind, owner.Name, ownerIsController}, + Value: 1, + }) + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ), } ) @@ -161,21 +219,22 @@ func wrapReplicationControllerFunc(f func(*v1.ReplicationController) *metric.Fam metricFamily := f(replicationController) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descReplicationControllerLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{replicationController.Namespace, replicationController.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descReplicationControllerLabelsDefaultLabels, []string{replicationController.Namespace, replicationController.Name}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createReplicationControllerListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createReplicationControllerListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.CoreV1().ReplicationControllers(ns).List(opts) + opts.FieldSelector = fieldSelector + return kubeClient.CoreV1().ReplicationControllers(ns).List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.CoreV1().ReplicationControllers(ns).Watch(opts) + opts.FieldSelector = fieldSelector + return kubeClient.CoreV1().ReplicationControllers(ns).Watch(context.TODO(), opts) }, } } diff --git a/internal/store/replicationcontroller_test.go b/internal/store/replicationcontroller_test.go index f3e3c33468..aea8bb1bb5 100644 --- a/internal/store/replicationcontroller_test.go +++ b/internal/store/replicationcontroller_test.go @@ -23,33 +23,38 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) var ( rc1Replicas int32 = 5 rc2Replicas int32 + rc3Replicas int32 ) func TestReplicationControllerStore(t *testing.T) { + var trueValue = true + // Fixed metadata on type and help text. We prepend this to every expected // output so we only have to modify a single place when doing adjustments. const metadata = ` - # HELP kube_replicationcontroller_created Unix creation timestamp + # HELP kube_replicationcontroller_created [STABLE] Unix creation timestamp # TYPE kube_replicationcontroller_created gauge - # HELP kube_replicationcontroller_metadata_generation Sequence number representing a specific generation of the desired state. + # HELP kube_replicationcontroller_metadata_generation [STABLE] Sequence number representing a specific generation of the desired state. # TYPE kube_replicationcontroller_metadata_generation gauge - # HELP kube_replicationcontroller_status_replicas The number of replicas per ReplicationController. + # HELP kube_replicationcontroller_owner Information about the ReplicationController's owner. + # TYPE kube_replicationcontroller_owner gauge + # HELP kube_replicationcontroller_status_replicas [STABLE] The number of replicas per ReplicationController. # TYPE kube_replicationcontroller_status_replicas gauge - # HELP kube_replicationcontroller_status_fully_labeled_replicas The number of fully labeled replicas per ReplicationController. + # HELP kube_replicationcontroller_status_fully_labeled_replicas [STABLE] The number of fully labeled replicas per ReplicationController. # TYPE kube_replicationcontroller_status_fully_labeled_replicas gauge - # HELP kube_replicationcontroller_status_available_replicas The number of available replicas per ReplicationController. + # HELP kube_replicationcontroller_status_available_replicas [STABLE] The number of available replicas per ReplicationController. # TYPE kube_replicationcontroller_status_available_replicas gauge - # HELP kube_replicationcontroller_status_ready_replicas The number of ready replicas per ReplicationController. + # HELP kube_replicationcontroller_status_ready_replicas [STABLE] The number of ready replicas per ReplicationController. # TYPE kube_replicationcontroller_status_ready_replicas gauge - # HELP kube_replicationcontroller_status_observed_generation The generation observed by the ReplicationController controller. + # HELP kube_replicationcontroller_status_observed_generation [STABLE] The generation observed by the ReplicationController controller. # TYPE kube_replicationcontroller_status_observed_generation gauge - # HELP kube_replicationcontroller_spec_replicas Number of desired pods for a ReplicationController. + # HELP kube_replicationcontroller_spec_replicas [STABLE] Number of desired pods for a ReplicationController. # TYPE kube_replicationcontroller_spec_replicas gauge ` cases := []generateMetricsTestCase{ @@ -60,6 +65,13 @@ func TestReplicationControllerStore(t *testing.T) { CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, Namespace: "ns1", Generation: 21, + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "DeploymentConfig", + Name: "dc-name", + Controller: &trueValue, + }, + }, }, Status: v1.ReplicationControllerStatus{ Replicas: 5, @@ -75,6 +87,7 @@ func TestReplicationControllerStore(t *testing.T) { Want: metadata + ` kube_replicationcontroller_created{namespace="ns1",replicationcontroller="rc1"} 1.5e+09 kube_replicationcontroller_metadata_generation{namespace="ns1",replicationcontroller="rc1"} 21 + kube_replicationcontroller_owner{namespace="ns1",owner_is_controller="true",owner_kind="DeploymentConfig",owner_name="dc-name",replicationcontroller="rc1"} 1 kube_replicationcontroller_status_replicas{namespace="ns1",replicationcontroller="rc1"} 5 kube_replicationcontroller_status_observed_generation{namespace="ns1",replicationcontroller="rc1"} 1 kube_replicationcontroller_status_fully_labeled_replicas{namespace="ns1",replicationcontroller="rc1"} 10 @@ -103,18 +116,55 @@ func TestReplicationControllerStore(t *testing.T) { }, Want: metadata + ` kube_replicationcontroller_metadata_generation{namespace="ns2",replicationcontroller="rc2"} 14 + kube_replicationcontroller_owner{namespace="ns2",owner_is_controller="",owner_kind="",owner_name="",replicationcontroller="rc2"} 1 kube_replicationcontroller_status_replicas{namespace="ns2",replicationcontroller="rc2"} 0 kube_replicationcontroller_status_observed_generation{namespace="ns2",replicationcontroller="rc2"} 5 kube_replicationcontroller_status_fully_labeled_replicas{namespace="ns2",replicationcontroller="rc2"} 5 kube_replicationcontroller_status_ready_replicas{namespace="ns2",replicationcontroller="rc2"} 0 kube_replicationcontroller_status_available_replicas{namespace="ns2",replicationcontroller="rc2"} 0 kube_replicationcontroller_spec_replicas{namespace="ns2",replicationcontroller="rc2"} 0 +`, + }, + { + Obj: &v1.ReplicationController{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rc3", + Namespace: "ns3", + Generation: 5, + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "DeploymentConfig", + Name: "dc-test", + Controller: nil, + }, + }, + }, + Status: v1.ReplicationControllerStatus{ + Replicas: 1, + FullyLabeledReplicas: 5, + ReadyReplicas: 2, + AvailableReplicas: 1, + ObservedGeneration: 1, + }, + Spec: v1.ReplicationControllerSpec{ + Replicas: &rc3Replicas, + }, + }, + Want: metadata + ` + kube_replicationcontroller_metadata_generation{namespace="ns3",replicationcontroller="rc3"} 5 + kube_replicationcontroller_owner{namespace="ns3",owner_is_controller="false",owner_kind="DeploymentConfig",owner_name="dc-test",replicationcontroller="rc3"} 1 + kube_replicationcontroller_status_replicas{namespace="ns3",replicationcontroller="rc3"} 1 + kube_replicationcontroller_status_observed_generation{namespace="ns3",replicationcontroller="rc3"} 1 + kube_replicationcontroller_status_fully_labeled_replicas{namespace="ns3",replicationcontroller="rc3"} 5 + kube_replicationcontroller_status_ready_replicas{namespace="ns3",replicationcontroller="rc3"} 2 + kube_replicationcontroller_status_available_replicas{namespace="ns3",replicationcontroller="rc3"} 1 + kube_replicationcontroller_spec_replicas{namespace="ns3",replicationcontroller="rc3"} 0 `, }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(replicationControllerMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(replicationControllerMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(replicationControllerMetricFamilies) + c.Headers = generator.ExtractMetricFamilyHeaders(replicationControllerMetricFamilies) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/resourcequota.go b/internal/store/resourcequota.go index 694290928b..fe55f98f0b 100644 --- a/internal/store/resourcequota.go +++ b/internal/store/resourcequota.go @@ -17,25 +17,31 @@ limitations under the License. package store import ( + "context" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" - "k8s.io/kube-state-metrics/pkg/metric" + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) var ( descResourceQuotaLabelsDefaultLabels = []string{"namespace", "resourcequota"} - resourceQuotaMetricFamilies = []metric.FamilyGenerator{ - { - Name: "kube_resourcequota_created", - Type: metric.Gauge, - Help: "Unix creation timestamp", - GenerateFunc: wrapResourceQuotaFunc(func(r *v1.ResourceQuota) *metric.Family { + resourceQuotaMetricFamilies = []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_resourcequota_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.STABLE, + "", + wrapResourceQuotaFunc(func(r *v1.ResourceQuota) *metric.Family { ms := []*metric.Metric{} if !r.CreationTimestamp.IsZero() { @@ -49,12 +55,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_resourcequota", - Type: metric.Gauge, - Help: "Information about resource quota.", - GenerateFunc: wrapResourceQuotaFunc(func(r *v1.ResourceQuota) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_resourcequota", + "Information about resource quota.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapResourceQuotaFunc(func(r *v1.ResourceQuota) *metric.Family { ms := []*metric.Metric{} for res, qty := range r.Status.Hard { @@ -78,7 +86,7 @@ var ( Metrics: ms, } }), - }, + ), } ) @@ -89,21 +97,22 @@ func wrapResourceQuotaFunc(f func(*v1.ResourceQuota) *metric.Family) func(interf metricFamily := f(resourceQuota) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descResourceQuotaLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{resourceQuota.Namespace, resourceQuota.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descResourceQuotaLabelsDefaultLabels, []string{resourceQuota.Namespace, resourceQuota.Name}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createResourceQuotaListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createResourceQuotaListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.CoreV1().ResourceQuotas(ns).List(opts) + opts.FieldSelector = fieldSelector + return kubeClient.CoreV1().ResourceQuotas(ns).List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.CoreV1().ResourceQuotas(ns).Watch(opts) + opts.FieldSelector = fieldSelector + return kubeClient.CoreV1().ResourceQuotas(ns).Watch(context.TODO(), opts) }, } } diff --git a/internal/store/resourcequota_test.go b/internal/store/resourcequota_test.go index f11e99719b..f342065cc3 100644 --- a/internal/store/resourcequota_test.go +++ b/internal/store/resourcequota_test.go @@ -24,16 +24,16 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) func TestResourceQuotaStore(t *testing.T) { // Fixed metadata on type and help text. We prepend this to every expected // output so we only have to modify a single place when doing adjustments. const metadata = ` - # HELP kube_resourcequota Information about resource quota. + # HELP kube_resourcequota [STABLE] Information about resource quota. # TYPE kube_resourcequota gauge - # HELP kube_resourcequota_created Unix creation timestamp + # HELP kube_resourcequota_created [STABLE] Unix creation timestamp # TYPE kube_resourcequota_created gauge ` cases := []generateMetricsTestCase{ @@ -134,8 +134,8 @@ func TestResourceQuotaStore(t *testing.T) { }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(resourceQuotaMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(resourceQuotaMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(resourceQuotaMetricFamilies) + c.Headers = generator.ExtractMetricFamilyHeaders(resourceQuotaMetricFamilies) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/role.go b/internal/store/role.go new file mode 100644 index 0000000000..eed2652d05 --- /dev/null +++ b/internal/store/role.go @@ -0,0 +1,160 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "context" + + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +var ( + descRoleAnnotationsName = "kube_role_annotations" + descRoleAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." + descRoleLabelsName = "kube_role_labels" + descRoleLabelsHelp = "Kubernetes labels converted to Prometheus labels." + descRoleLabelsDefaultLabels = []string{"namespace", "role"} +) + +func roleMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + descRoleAnnotationsName, + descRoleAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapRoleFunc(func(r *rbacv1.Role) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", r.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descRoleLabelsName, + descRoleLabelsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapRoleFunc(func(r *rbacv1.Role) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", r.Labels, allowLabelsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: labelKeys, + LabelValues: labelValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_role_info", + "Information about role.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapRoleFunc(func(r *rbacv1.Role) *metric.Family { + return &metric.Family{ + Metrics: []*metric.Metric{{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: 1, + }}, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_role_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapRoleFunc(func(r *rbacv1.Role) *metric.Family { + ms := []*metric.Metric{} + + if !r.CreationTimestamp.IsZero() { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64(r.CreationTimestamp.Unix()), + }) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_role_metadata_resource_version", + "Resource version representing a specific version of the role.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapRoleFunc(func(r *rbacv1.Role) *metric.Family { + return &metric.Family{ + Metrics: resourceVersionMetric(r.ObjectMeta.ResourceVersion), + } + }), + ), + } +} + +func createRoleListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { + return &cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + opts.FieldSelector = fieldSelector + return kubeClient.RbacV1().Roles(ns).List(context.TODO(), opts) + }, + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + opts.FieldSelector = fieldSelector + return kubeClient.RbacV1().Roles(ns).Watch(context.TODO(), opts) + }, + } +} + +func wrapRoleFunc(f func(*rbacv1.Role) *metric.Family) func(interface{}) *metric.Family { + return func(obj interface{}) *metric.Family { + role := obj.(*rbacv1.Role) + + metricFamily := f(role) + + for _, m := range metricFamily.Metrics { + m.LabelKeys, m.LabelValues = mergeKeyValues(descRoleLabelsDefaultLabels, []string{role.Namespace, role.Name}, m.LabelKeys, m.LabelValues) + } + + return metricFamily + } +} diff --git a/internal/store/role_test.go b/internal/store/role_test.go new file mode 100644 index 0000000000..499028aea2 --- /dev/null +++ b/internal/store/role_test.go @@ -0,0 +1,105 @@ +/* +Copyright 2012 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "testing" + + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +func TestRoleStore(t *testing.T) { + startTime := 1501569018 + metav1StartTime := metav1.Unix(int64(startTime), 0) + + cases := []generateMetricsTestCase{ + { + AllowAnnotationsList: []string{ + "app.k8s.io/owner", + }, + AllowLabelsList: []string{ + "app", + }, + Obj: &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: "role1", + Namespace: "ns1", + ResourceVersion: "BBBBB", + Annotations: map[string]string{ + "app": "mysql-server", + "app.k8s.io/owner": "@foo", + }, + Labels: map[string]string{ + "excluded": "me", + "app": "mysql-server", + }, + }, + }, + Want: ` + # HELP kube_role_annotations Kubernetes annotations converted to Prometheus labels. + # HELP kube_role_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_role_info Information about role. + # HELP kube_role_metadata_resource_version Resource version representing a specific version of the role. + # TYPE kube_role_annotations gauge + # TYPE kube_role_labels gauge + # TYPE kube_role_info gauge + # TYPE kube_role_metadata_resource_version gauge + kube_role_annotations{annotation_app_k8s_io_owner="@foo",role="role1",namespace="ns1"} 1 + kube_role_labels{role="role1",label_app="mysql-server",namespace="ns1"} 1 + kube_role_info{role="role1",namespace="ns1"} 1 +`, + MetricNames: []string{ + "kube_role_annotations", + "kube_role_labels", + "kube_role_info", + "kube_role_metadata_resource_version", + }, + }, + { + Obj: &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: "role2", + Namespace: "ns2", + CreationTimestamp: metav1StartTime, + ResourceVersion: "10596", + }, + }, + Want: ` + # HELP kube_role_created Unix creation timestamp + # HELP kube_role_info Information about role. + # HELP kube_role_metadata_resource_version Resource version representing a specific version of the role. + # TYPE kube_role_created gauge + # TYPE kube_role_info gauge + # TYPE kube_role_metadata_resource_version gauge + kube_role_info{role="role2",namespace="ns2"} 1 + kube_role_created{role="role2",namespace="ns2"} 1.501569018e+09 + kube_role_metadata_resource_version{role="role2",namespace="ns2"} 10596 + `, + MetricNames: []string{"kube_role_info", "kube_role_created", "kube_role_metadata_resource_version"}, + }, + } + for i, c := range cases { + c.Func = generator.ComposeMetricGenFuncs(roleMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) + c.Headers = generator.ExtractMetricFamilyHeaders(roleMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) + if err := c.run(); err != nil { + t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) + } + } +} diff --git a/internal/store/rolebinding.go b/internal/store/rolebinding.go new file mode 100644 index 0000000000..beca54c459 --- /dev/null +++ b/internal/store/rolebinding.go @@ -0,0 +1,162 @@ +/* +Copyright 2022 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "context" + + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +var ( + descRoleBindingAnnotationsName = "kube_rolebinding_annotations" + descRoleBindingAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." + descRoleBindingLabelsName = "kube_rolebinding_labels" + descRoleBindingLabelsHelp = "Kubernetes labels converted to Prometheus labels." + descRoleBindingLabelsDefaultLabels = []string{"namespace", "rolebinding"} +) + +func roleBindingMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + descRoleBindingAnnotationsName, + descRoleBindingAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapRoleBindingFunc(func(r *rbacv1.RoleBinding) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", r.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descRoleBindingLabelsName, + descRoleBindingLabelsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapRoleBindingFunc(func(r *rbacv1.RoleBinding) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", r.Labels, allowLabelsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: labelKeys, + LabelValues: labelValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_rolebinding_info", + "Information about rolebinding.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapRoleBindingFunc(func(r *rbacv1.RoleBinding) *metric.Family { + labelKeys := []string{"roleref_kind", "roleref_name"} + labelValues := []string{r.RoleRef.Kind, r.RoleRef.Name} + return &metric.Family{ + Metrics: []*metric.Metric{{ + LabelKeys: labelKeys, + LabelValues: labelValues, + Value: 1, + }}, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_rolebinding_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapRoleBindingFunc(func(r *rbacv1.RoleBinding) *metric.Family { + ms := []*metric.Metric{} + + if !r.CreationTimestamp.IsZero() { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64(r.CreationTimestamp.Unix()), + }) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_rolebinding_metadata_resource_version", + "Resource version representing a specific version of the rolebinding.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapRoleBindingFunc(func(r *rbacv1.RoleBinding) *metric.Family { + return &metric.Family{ + Metrics: resourceVersionMetric(r.ObjectMeta.ResourceVersion), + } + }), + ), + } +} + +func createRoleBindingListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { + return &cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + opts.FieldSelector = fieldSelector + return kubeClient.RbacV1().RoleBindings(ns).List(context.TODO(), opts) + }, + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + opts.FieldSelector = fieldSelector + return kubeClient.RbacV1().RoleBindings(ns).Watch(context.TODO(), opts) + }, + } +} + +func wrapRoleBindingFunc(f func(*rbacv1.RoleBinding) *metric.Family) func(interface{}) *metric.Family { + return func(obj interface{}) *metric.Family { + rolebinding := obj.(*rbacv1.RoleBinding) + + metricFamily := f(rolebinding) + + for _, m := range metricFamily.Metrics { + m.LabelKeys, m.LabelValues = mergeKeyValues(descRoleBindingLabelsDefaultLabels, []string{rolebinding.Namespace, rolebinding.Name}, m.LabelKeys, m.LabelValues) + } + + return metricFamily + } +} diff --git a/internal/store/rolebinding_test.go b/internal/store/rolebinding_test.go new file mode 100644 index 0000000000..17ce4848cd --- /dev/null +++ b/internal/store/rolebinding_test.go @@ -0,0 +1,115 @@ +/* +Copyright 2022 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "testing" + + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +func TestRoleBindingStore(t *testing.T) { + startTime := 1501569018 + metav1StartTime := metav1.Unix(int64(startTime), 0) + + cases := []generateMetricsTestCase{ + { + AllowAnnotationsList: []string{ + "app.k8s.io/owner", + }, + AllowLabelsList: []string{ + "app", + }, + Obj: &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rolebinding1", + Namespace: "ns1", + ResourceVersion: "BBBBB", + Annotations: map[string]string{ + "app": "mysql-server", + "app.k8s.io/owner": "@foo", + }, + Labels: map[string]string{ + "excluded": "me", + "app": "mysql-server", + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "role", + }, + }, + Want: ` + # HELP kube_rolebinding_annotations Kubernetes annotations converted to Prometheus labels. + # HELP kube_rolebinding_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_rolebinding_info Information about rolebinding. + # HELP kube_rolebinding_metadata_resource_version Resource version representing a specific version of the rolebinding. + # TYPE kube_rolebinding_annotations gauge + # TYPE kube_rolebinding_labels gauge + # TYPE kube_rolebinding_info gauge + # TYPE kube_rolebinding_metadata_resource_version gauge + kube_rolebinding_annotations{annotation_app_k8s_io_owner="@foo",rolebinding="rolebinding1",namespace="ns1"} 1 + kube_rolebinding_labels{rolebinding="rolebinding1",label_app="mysql-server",namespace="ns1"} 1 + kube_rolebinding_info{rolebinding="rolebinding1",namespace="ns1",roleref_kind="Role",roleref_name="role"} 1 +`, + MetricNames: []string{ + "kube_rolebinding_annotations", + "kube_rolebinding_labels", + "kube_rolebinding_info", + "kube_rolebinding_metadata_resource_version", + }, + }, + { + Obj: &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rolebinding2", + Namespace: "ns2", + CreationTimestamp: metav1StartTime, + ResourceVersion: "10596", + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "role", + }, + }, + Want: ` + # HELP kube_rolebinding_created Unix creation timestamp + # HELP kube_rolebinding_info Information about rolebinding. + # HELP kube_rolebinding_metadata_resource_version Resource version representing a specific version of the rolebinding. + # TYPE kube_rolebinding_created gauge + # TYPE kube_rolebinding_info gauge + # TYPE kube_rolebinding_metadata_resource_version gauge + kube_rolebinding_info{rolebinding="rolebinding2",namespace="ns2",roleref_kind="Role",roleref_name="role"} 1 + kube_rolebinding_created{rolebinding="rolebinding2",namespace="ns2"} 1.501569018e+09 + kube_rolebinding_metadata_resource_version{rolebinding="rolebinding2",namespace="ns2"} 10596 + `, + MetricNames: []string{"kube_rolebinding_info", "kube_rolebinding_created", "kube_rolebinding_metadata_resource_version"}, + }, + } + for i, c := range cases { + c.Func = generator.ComposeMetricGenFuncs(roleBindingMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) + c.Headers = generator.ExtractMetricFamilyHeaders(roleBindingMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) + if err := c.run(); err != nil { + t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) + } + } +} diff --git a/internal/store/secret.go b/internal/store/secret.go index c501bda148..50572f6bc9 100644 --- a/internal/store/secret.go +++ b/internal/store/secret.go @@ -17,27 +17,37 @@ limitations under the License. package store import ( + "context" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" - "k8s.io/kube-state-metrics/pkg/metric" + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) var ( + descSecretAnnotationsName = "kube_secret_annotations" + descSecretAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." //nolint:gosec descSecretLabelsName = "kube_secret_labels" - descSecretLabelsHelp = "Kubernetes labels converted to Prometheus labels." + descSecretLabelsHelp = "Kubernetes labels converted to Prometheus labels." //nolint:gosec descSecretLabelsDefaultLabels = []string{"namespace", "secret"} +) - secretMetricFamilies = []metric.FamilyGenerator{ - { - Name: "kube_secret_info", - Type: metric.Gauge, - Help: "Information about secret.", - GenerateFunc: wrapSecretFunc(func(s *v1.Secret) *metric.Family { +func secretMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_secret_info", + "Information about secret.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapSecretFunc(func(s *v1.Secret) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -46,12 +56,14 @@ var ( }, } }), - }, - { - Name: "kube_secret_type", - Type: metric.Gauge, - Help: "Type about secret.", - GenerateFunc: wrapSecretFunc(func(s *v1.Secret) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_secret_type", + "Type about secret.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapSecretFunc(func(s *v1.Secret) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -62,13 +74,35 @@ var ( }, } }), - }, - { - Name: descSecretLabelsName, - Type: metric.Gauge, - Help: descSecretLabelsHelp, - GenerateFunc: wrapSecretFunc(func(s *v1.Secret) *metric.Family { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(s.Labels) + ), + *generator.NewFamilyGeneratorWithStability( + descSecretAnnotationsName, + descSecretAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapSecretFunc(func(s *v1.Secret) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", s.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + + }), + ), + *generator.NewFamilyGeneratorWithStability( + descSecretLabelsName, + descSecretLabelsHelp, + metric.Gauge, + basemetrics.STABLE, + "", + wrapSecretFunc(func(s *v1.Secret) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", s.Labels, allowLabelsList) return &metric.Family{ Metrics: []*metric.Metric{ { @@ -80,12 +114,14 @@ var ( } }), - }, - { - Name: "kube_secret_created", - Type: metric.Gauge, - Help: "Unix creation timestamp", - GenerateFunc: wrapSecretFunc(func(s *v1.Secret) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_secret_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.STABLE, + "", + wrapSecretFunc(func(s *v1.Secret) *metric.Family { ms := []*metric.Metric{} if !s.CreationTimestamp.IsZero() { @@ -98,19 +134,22 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_secret_metadata_resource_version", - Type: metric.Gauge, - Help: "Resource version representing a specific version of secret.", - GenerateFunc: wrapSecretFunc(func(s *v1.Secret) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_secret_metadata_resource_version", + "Resource version representing a specific version of secret.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapSecretFunc(func(s *v1.Secret) *metric.Family { return &metric.Family{ Metrics: resourceVersionMetric(s.ObjectMeta.ResourceVersion), } }), - }, + ), } -) + +} func wrapSecretFunc(f func(*v1.Secret) *metric.Family) func(interface{}) *metric.Family { return func(obj interface{}) *metric.Family { @@ -119,21 +158,22 @@ func wrapSecretFunc(f func(*v1.Secret) *metric.Family) func(interface{}) *metric metricFamily := f(secret) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descSecretLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{secret.Namespace, secret.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descSecretLabelsDefaultLabels, []string{secret.Namespace, secret.Name}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createSecretListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createSecretListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.CoreV1().Secrets(ns).List(opts) + opts.FieldSelector = fieldSelector + return kubeClient.CoreV1().Secrets(ns).List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.CoreV1().Secrets(ns).Watch(opts) + opts.FieldSelector = fieldSelector + return kubeClient.CoreV1().Secrets(ns).Watch(context.TODO(), opts) }, } } diff --git a/internal/store/secret_test.go b/internal/store/secret_test.go index 2367afdf76..bb10cc6107 100644 --- a/internal/store/secret_test.go +++ b/internal/store/secret_test.go @@ -22,7 +22,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) func TestSecretStore(t *testing.T) { @@ -39,11 +39,11 @@ func TestSecretStore(t *testing.T) { Type: v1.SecretTypeOpaque, }, Want: ` - # HELP kube_secret_created Unix creation timestamp - # HELP kube_secret_info Information about secret. - # HELP kube_secret_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_secret_created [STABLE] Unix creation timestamp + # HELP kube_secret_info [STABLE] Information about secret. + # HELP kube_secret_labels [STABLE] Kubernetes labels converted to Prometheus labels. # HELP kube_secret_metadata_resource_version Resource version representing a specific version of secret. - # HELP kube_secret_type Type about secret. + # HELP kube_secret_type [STABLE] Type about secret. # TYPE kube_secret_created gauge # TYPE kube_secret_info gauge # TYPE kube_secret_labels gauge @@ -67,11 +67,11 @@ func TestSecretStore(t *testing.T) { Type: v1.SecretTypeServiceAccountToken, }, Want: ` - # HELP kube_secret_created Unix creation timestamp - # HELP kube_secret_info Information about secret. - # HELP kube_secret_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_secret_created [STABLE] Unix creation timestamp + # HELP kube_secret_info [STABLE] Information about secret. + # HELP kube_secret_labels [STABLE] Kubernetes labels converted to Prometheus labels. # HELP kube_secret_metadata_resource_version Resource version representing a specific version of secret. - # HELP kube_secret_type Type about secret. + # HELP kube_secret_type [STABLE] Type about secret. # TYPE kube_secret_created gauge # TYPE kube_secret_info gauge # TYPE kube_secret_labels gauge @@ -96,11 +96,11 @@ func TestSecretStore(t *testing.T) { Type: v1.SecretTypeDockercfg, }, Want: ` - # HELP kube_secret_created Unix creation timestamp - # HELP kube_secret_info Information about secret. - # HELP kube_secret_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_secret_created [STABLE] Unix creation timestamp + # HELP kube_secret_info [STABLE] Information about secret. + # HELP kube_secret_labels [STABLE] Kubernetes labels converted to Prometheus labels. # HELP kube_secret_metadata_resource_version Resource version representing a specific version of secret. - # HELP kube_secret_type Type about secret. + # HELP kube_secret_type [STABLE] Type about secret. # TYPE kube_secret_created gauge # TYPE kube_secret_info gauge # TYPE kube_secret_labels gauge @@ -110,14 +110,14 @@ func TestSecretStore(t *testing.T) { kube_secret_type{namespace="ns3",secret="secret3",type="kubernetes.io/dockercfg"} 1 kube_secret_created{namespace="ns3",secret="secret3"} 1.501569018e+09 kube_secret_metadata_resource_version{namespace="ns3",secret="secret3"} 0 - kube_secret_labels{label_test_3="test-3",namespace="ns3",secret="secret3"} 1 + kube_secret_labels{namespace="ns3",secret="secret3"} 1 `, MetricNames: []string{"kube_secret_info", "kube_secret_metadata_resource_version", "kube_secret_created", "kube_secret_labels", "kube_secret_type"}, }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(secretMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(secretMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(secretMetricFamilies(nil, nil)) + c.Headers = generator.ExtractMetricFamilyHeaders(secretMetricFamilies(nil, nil)) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/service.go b/internal/store/service.go index 21d2bfe4e4..d0f4c552a4 100644 --- a/internal/store/service.go +++ b/internal/store/service.go @@ -17,27 +17,37 @@ limitations under the License. package store import ( + "context" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" - "k8s.io/kube-state-metrics/pkg/metric" + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) var ( + descServiceAnnotationsName = "kube_service_annotations" + descServiceAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." descServiceLabelsName = "kube_service_labels" descServiceLabelsHelp = "Kubernetes labels converted to Prometheus labels." - descServiceLabelsDefaultLabels = []string{"namespace", "service"} - - serviceMetricFamilies = []metric.FamilyGenerator{ - { - Name: "kube_service_info", - Type: metric.Gauge, - Help: "Information about service.", - GenerateFunc: wrapSvcFunc(func(s *v1.Service) *metric.Family { + descServiceLabelsDefaultLabels = []string{"namespace", "service", "uid"} +) + +func serviceMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_service_info", + "Information about service.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapSvcFunc(func(s *v1.Service) *metric.Family { m := metric.Metric{ LabelKeys: []string{"cluster_ip", "external_name", "load_balancer_ip"}, LabelValues: []string{s.Spec.ClusterIP, s.Spec.ExternalName, s.Spec.LoadBalancerIP}, @@ -45,12 +55,14 @@ var ( } return &metric.Family{Metrics: []*metric.Metric{&m}} }), - }, - { - Name: "kube_service_created", - Type: metric.Gauge, - Help: "Unix creation timestamp", - GenerateFunc: wrapSvcFunc(func(s *v1.Service) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_service_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.STABLE, + "", + wrapSvcFunc(func(s *v1.Service) *metric.Family { if !s.CreationTimestamp.IsZero() { m := metric.Metric{ LabelKeys: nil, @@ -61,12 +73,14 @@ var ( } return &metric.Family{Metrics: []*metric.Metric{}} }), - }, - { - Name: "kube_service_spec_type", - Type: metric.Gauge, - Help: "Type about service.", - GenerateFunc: wrapSvcFunc(func(s *v1.Service) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_service_spec_type", + "Type about service.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapSvcFunc(func(s *v1.Service) *metric.Family { m := metric.Metric{ LabelKeys: []string{"type"}, @@ -75,27 +89,46 @@ var ( } return &metric.Family{Metrics: []*metric.Metric{&m}} }), - }, - { - Name: descServiceLabelsName, - Type: metric.Gauge, - Help: descServiceLabelsHelp, - GenerateFunc: wrapSvcFunc(func(s *v1.Service) *metric.Family { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(s.Labels) + ), + *generator.NewFamilyGeneratorWithStability( + descServiceAnnotationsName, + descServiceAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapSvcFunc(func(s *v1.Service) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", s.Annotations, allowAnnotationsList) + m := metric.Metric{ + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + } + return &metric.Family{Metrics: []*metric.Metric{&m}} + }), + ), + *generator.NewFamilyGeneratorWithStability( + descServiceLabelsName, + descServiceLabelsHelp, + metric.Gauge, + basemetrics.STABLE, + "", + wrapSvcFunc(func(s *v1.Service) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", s.Labels, allowLabelsList) m := metric.Metric{ - LabelKeys: labelKeys, LabelValues: labelValues, Value: 1, } return &metric.Family{Metrics: []*metric.Metric{&m}} }), - }, - { - Name: "kube_service_spec_external_ip", - Type: metric.Gauge, - Help: "Service external ips. One series for each ip", - GenerateFunc: wrapSvcFunc(func(s *v1.Service) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_service_spec_external_ip", + "Service external ips. One series for each ip", + metric.Gauge, + basemetrics.STABLE, + "", + wrapSvcFunc(func(s *v1.Service) *metric.Family { if len(s.Spec.ExternalIPs) == 0 { return &metric.Family{ Metrics: []*metric.Metric{}, @@ -116,12 +149,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_service_status_load_balancer_ingress", - Type: metric.Gauge, - Help: "Service load balancer ingress status", - GenerateFunc: wrapSvcFunc(func(s *v1.Service) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_service_status_load_balancer_ingress", + "Service load balancer ingress status", + metric.Gauge, + basemetrics.STABLE, + "", + wrapSvcFunc(func(s *v1.Service) *metric.Family { if len(s.Status.LoadBalancer.Ingress) == 0 { return &metric.Family{ Metrics: []*metric.Metric{}, @@ -142,9 +177,9 @@ var ( Metrics: ms, } }), - }, + ), } -) +} func wrapSvcFunc(f func(*v1.Service) *metric.Family) func(interface{}) *metric.Family { return func(obj interface{}) *metric.Family { @@ -153,21 +188,22 @@ func wrapSvcFunc(f func(*v1.Service) *metric.Family) func(interface{}) *metric.F metricFamily := f(svc) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descServiceLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{svc.Namespace, svc.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descServiceLabelsDefaultLabels, []string{svc.Namespace, svc.Name, string(svc.UID)}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createServiceListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createServiceListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.CoreV1().Services(ns).List(opts) + opts.FieldSelector = fieldSelector + return kubeClient.CoreV1().Services(ns).List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.CoreV1().Services(ns).Watch(opts) + opts.FieldSelector = fieldSelector + return kubeClient.CoreV1().Services(ns).Watch(context.TODO(), opts) }, } } diff --git a/internal/store/service_test.go b/internal/store/service_test.go index 1dc3f00d69..c291a48a08 100644 --- a/internal/store/service_test.go +++ b/internal/store/service_test.go @@ -23,24 +23,26 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) func TestServiceStore(t *testing.T) { // Fixed metadata on type and help text. We prepend this to every expected // output so we only have to modify a single place when doing adjustments. const metadata = ` - # HELP kube_service_info Information about service. + # HELP kube_service_annotations Kubernetes annotations converted to Prometheus labels. + # TYPE kube_service_annotations gauge + # HELP kube_service_info [STABLE] Information about service. # TYPE kube_service_info gauge - # HELP kube_service_created Unix creation timestamp + # HELP kube_service_created [STABLE] Unix creation timestamp # TYPE kube_service_created gauge - # HELP kube_service_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_service_labels [STABLE] Kubernetes labels converted to Prometheus labels. # TYPE kube_service_labels gauge - # HELP kube_service_spec_type Type about service. + # HELP kube_service_spec_type [STABLE] Type about service. # TYPE kube_service_spec_type gauge - # HELP kube_service_spec_external_ip Service external ips. One series for each ip + # HELP kube_service_spec_external_ip [STABLE] Service external ips. One series for each ip # TYPE kube_service_spec_external_ip gauge - # HELP kube_service_status_load_balancer_ingress Service load balancer ingress status + # HELP kube_service_status_load_balancer_ingress [STABLE] Service load balancer ingress status # TYPE kube_service_status_load_balancer_ingress gauge ` cases := []generateMetricsTestCase{ @@ -50,6 +52,7 @@ func TestServiceStore(t *testing.T) { Name: "test-service1", CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, Namespace: "default", + UID: "uid1", Labels: map[string]string{ "app": "example1", }, @@ -60,20 +63,24 @@ func TestServiceStore(t *testing.T) { }, }, Want: ` - # HELP kube_service_created Unix creation timestamp - # HELP kube_service_info Information about service. - # HELP kube_service_labels Kubernetes labels converted to Prometheus labels. - # HELP kube_service_spec_type Type about service. + # HELP kube_service_annotations Kubernetes annotations converted to Prometheus labels. + # HELP kube_service_created [STABLE] Unix creation timestamp + # HELP kube_service_info [STABLE] Information about service. + # HELP kube_service_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # HELP kube_service_spec_type [STABLE] Type about service. + # TYPE kube_service_annotations gauge # TYPE kube_service_created gauge # TYPE kube_service_info gauge # TYPE kube_service_labels gauge # TYPE kube_service_spec_type gauge - kube_service_created{namespace="default",service="test-service1"} 1.5e+09 - kube_service_info{cluster_ip="1.2.3.4",external_name="",load_balancer_ip="",namespace="default",service="test-service1"} 1 - kube_service_labels{label_app="example1",namespace="default",service="test-service1"} 1 - kube_service_spec_type{namespace="default",service="test-service1",type="ClusterIP"} 1 + kube_service_annotations{namespace="default",service="test-service1",uid="uid1"} 1 + kube_service_created{namespace="default",service="test-service1",uid="uid1"} 1.5e+09 + kube_service_info{cluster_ip="1.2.3.4",external_name="",load_balancer_ip="",namespace="default",service="test-service1",uid="uid1"} 1 + kube_service_labels{namespace="default",service="test-service1",uid="uid1"} 1 + kube_service_spec_type{namespace="default",service="test-service1",type="ClusterIP",uid="uid1"} 1 `, MetricNames: []string{ + "kube_service_annotations", "kube_service_created", "kube_service_info", "kube_service_labels", @@ -87,6 +94,7 @@ func TestServiceStore(t *testing.T) { Name: "test-service2", CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, Namespace: "default", + UID: "uid2", Labels: map[string]string{ "app": "example2", }, @@ -97,10 +105,11 @@ func TestServiceStore(t *testing.T) { }, }, Want: metadata + ` - kube_service_created{namespace="default",service="test-service2"} 1.5e+09 - kube_service_info{cluster_ip="1.2.3.5",external_name="",load_balancer_ip="",namespace="default",service="test-service2"} 1 - kube_service_labels{label_app="example2",namespace="default",service="test-service2"} 1 - kube_service_spec_type{namespace="default",service="test-service2",type="NodePort"} 1 + kube_service_annotations{namespace="default",service="test-service2",uid="uid2"} 1 + kube_service_created{namespace="default",service="test-service2",uid="uid2"} 1.5e+09 + kube_service_info{cluster_ip="1.2.3.5",external_name="",load_balancer_ip="",namespace="default",service="test-service2",uid="uid2"} 1 + kube_service_labels{namespace="default",service="test-service2",uid="uid2"} 1 + kube_service_spec_type{namespace="default",service="test-service2",uid="uid2",type="NodePort"} 1 `, }, { @@ -109,6 +118,7 @@ func TestServiceStore(t *testing.T) { Name: "test-service3", CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, Namespace: "default", + UID: "uid3", Labels: map[string]string{ "app": "example3", }, @@ -120,10 +130,11 @@ func TestServiceStore(t *testing.T) { }, }, Want: metadata + ` - kube_service_created{namespace="default",service="test-service3"} 1.5e+09 - kube_service_info{cluster_ip="1.2.3.6",external_name="",load_balancer_ip="1.2.3.7",namespace="default",service="test-service3"} 1 - kube_service_labels{label_app="example3",namespace="default",service="test-service3"} 1 - kube_service_spec_type{namespace="default",service="test-service3",type="LoadBalancer"} 1 + kube_service_annotations{namespace="default",service="test-service3",uid="uid3"} 1 + kube_service_created{namespace="default",service="test-service3",uid="uid3"} 1.5e+09 + kube_service_info{cluster_ip="1.2.3.6",external_name="",load_balancer_ip="1.2.3.7",namespace="default",service="test-service3",uid="uid3"} 1 + kube_service_labels{namespace="default",service="test-service3",uid="uid3"} 1 + kube_service_spec_type{namespace="default",service="test-service3",type="LoadBalancer",uid="uid3"} 1 `, }, { @@ -132,6 +143,7 @@ func TestServiceStore(t *testing.T) { Name: "test-service4", CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, Namespace: "default", + UID: "uid4", Labels: map[string]string{ "app": "example4", }, @@ -142,10 +154,11 @@ func TestServiceStore(t *testing.T) { }, }, Want: metadata + ` - kube_service_created{namespace="default",service="test-service4"} 1.5e+09 - kube_service_info{cluster_ip="",external_name="www.example.com",load_balancer_ip="",namespace="default",service="test-service4"} 1 - kube_service_labels{label_app="example4",namespace="default",service="test-service4"} 1 - kube_service_spec_type{namespace="default",service="test-service4",type="ExternalName"} 1 + kube_service_annotations{namespace="default",service="test-service4",uid="uid4"} 1 + kube_service_created{namespace="default",service="test-service4",uid="uid4"} 1.5e+09 + kube_service_info{cluster_ip="",external_name="www.example.com",load_balancer_ip="",namespace="default",service="test-service4",uid="uid4"} 1 + kube_service_labels{namespace="default",service="test-service4",uid="uid4"} 1 + kube_service_spec_type{namespace="default",service="test-service4",uid="uid4",type="ExternalName"} 1 `, }, { @@ -154,6 +167,7 @@ func TestServiceStore(t *testing.T) { Name: "test-service5", CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, Namespace: "default", + UID: "uid5", Labels: map[string]string{ "app": "example5", }, @@ -173,11 +187,12 @@ func TestServiceStore(t *testing.T) { }, }, Want: metadata + ` - kube_service_created{namespace="default",service="test-service5"} 1.5e+09 - kube_service_info{cluster_ip="",external_name="",load_balancer_ip="",namespace="default",service="test-service5"} 1 - kube_service_labels{label_app="example5",namespace="default",service="test-service5"} 1 - kube_service_spec_type{namespace="default",service="test-service5",type="LoadBalancer"} 1 - kube_service_status_load_balancer_ingress{hostname="www.example.com",ip="1.2.3.8",namespace="default",service="test-service5"} 1 + kube_service_annotations{namespace="default",service="test-service5",uid="uid5"} 1 + kube_service_created{namespace="default",service="test-service5",uid="uid5"} 1.5e+09 + kube_service_info{cluster_ip="",external_name="",load_balancer_ip="",namespace="default",service="test-service5",uid="uid5"} 1 + kube_service_labels{namespace="default",service="test-service5",uid="uid5"} 1 + kube_service_spec_type{namespace="default",service="test-service5",type="LoadBalancer",uid="uid5"} 1 + kube_service_status_load_balancer_ingress{hostname="www.example.com",ip="1.2.3.8",namespace="default",service="test-service5",uid="uid5"} 1 `, }, { @@ -186,6 +201,7 @@ func TestServiceStore(t *testing.T) { Name: "test-service6", CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, Namespace: "default", + UID: "uid6", Labels: map[string]string{ "app": "example6", }, @@ -199,18 +215,19 @@ func TestServiceStore(t *testing.T) { }, }, Want: metadata + ` - kube_service_created{namespace="default",service="test-service6"} 1.5e+09 - kube_service_info{cluster_ip="",external_name="",load_balancer_ip="",namespace="default",service="test-service6"} 1 - kube_service_labels{label_app="example6",namespace="default",service="test-service6"} 1 - kube_service_spec_type{namespace="default",service="test-service6",type="ClusterIP"} 1 - kube_service_spec_external_ip{external_ip="1.2.3.9",namespace="default",service="test-service6"} 1 - kube_service_spec_external_ip{external_ip="1.2.3.10",namespace="default",service="test-service6"} 1 + kube_service_annotations{namespace="default",service="test-service6",uid="uid6"} 1 + kube_service_created{namespace="default",service="test-service6",uid="uid6"} 1.5e+09 + kube_service_info{cluster_ip="",external_name="",load_balancer_ip="",namespace="default",service="test-service6",uid="uid6"} 1 + kube_service_labels{namespace="default",service="test-service6",uid="uid6"} 1 + kube_service_spec_type{namespace="default",service="test-service6",uid="uid6",type="ClusterIP"} 1 + kube_service_spec_external_ip{external_ip="1.2.3.9",namespace="default",service="test-service6",uid="uid6"} 1 + kube_service_spec_external_ip{external_ip="1.2.3.10",namespace="default",service="test-service6",uid="uid6"} 1 `, }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(serviceMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(serviceMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(serviceMetricFamilies(nil, nil)) + c.Headers = generator.ExtractMetricFamilyHeaders(serviceMetricFamilies(nil, nil)) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/serviceaccount.go b/internal/store/serviceaccount.go new file mode 100644 index 0000000000..d17ce867b1 --- /dev/null +++ b/internal/store/serviceaccount.go @@ -0,0 +1,245 @@ +/* +Copyright 2022 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "context" + "strconv" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +var ( + descServiceAccountLabelsDefaultLabels = []string{"namespace", "serviceaccount", "uid"} +) + +func serviceAccountMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + createServiceAccountInfoFamilyGenerator(), + createServiceAccountCreatedFamilyGenerator(), + createServiceAccountDeletedFamilyGenerator(), + createServiceAccountSecretFamilyGenerator(), + createServiceAccountImagePullSecretFamilyGenerator(), + createServiceAccountAnnotationsGenerator(allowAnnotationsList), + createServiceAccountLabelsGenerator(allowLabelsList), + } +} + +func createServiceAccountInfoFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_serviceaccount_info", + "Information about a service account", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapServiceAccountFunc(func(sa *v1.ServiceAccount) *metric.Family { + var labelKeys []string + var labelValues []string + + if sa.AutomountServiceAccountToken != nil { + labelKeys = append(labelKeys, "automount_token") + labelValues = append(labelValues, strconv.FormatBool(*sa.AutomountServiceAccountToken)) + } + + return &metric.Family{ + Metrics: []*metric.Metric{{ + LabelKeys: labelKeys, + LabelValues: labelValues, + Value: 1, + }}, + } + }), + ) +} + +func createServiceAccountCreatedFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_serviceaccount_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapServiceAccountFunc(func(sa *v1.ServiceAccount) *metric.Family { + var ms []*metric.Metric + + if !sa.CreationTimestamp.IsZero() { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64(sa.CreationTimestamp.Unix()), + }) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createServiceAccountDeletedFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_serviceaccount_deleted", + "Unix deletion timestamp", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapServiceAccountFunc(func(sa *v1.ServiceAccount) *metric.Family { + var ms []*metric.Metric + + if sa.DeletionTimestamp != nil && !sa.DeletionTimestamp.IsZero() { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{}, + LabelValues: []string{}, + Value: float64(sa.DeletionTimestamp.Unix()), + }) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createServiceAccountSecretFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_serviceaccount_secret", + "Secret being referenced by a service account", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapServiceAccountFunc(func(sa *v1.ServiceAccount) *metric.Family { + var ms []*metric.Metric + + for _, s := range sa.Secrets { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"name"}, + LabelValues: []string{s.Name}, + Value: 1, + }) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createServiceAccountImagePullSecretFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_serviceaccount_image_pull_secret", + "Secret being referenced by a service account for the purpose of pulling images", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapServiceAccountFunc(func(sa *v1.ServiceAccount) *metric.Family { + var ms []*metric.Metric + + for _, s := range sa.ImagePullSecrets { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"name"}, + LabelValues: []string{s.Name}, + Value: 1, + }) + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + +func createServiceAccountAnnotationsGenerator(allowAnnotations []string) generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_serviceaccount_annotations", + "Kubernetes annotations converted to Prometheus labels.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapServiceAccountFunc(func(sa *v1.ServiceAccount) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", sa.Annotations, allowAnnotations) + m := metric.Metric{ + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + } + return &metric.Family{ + Metrics: []*metric.Metric{&m}, + } + }), + ) +} + +func createServiceAccountLabelsGenerator(allowLabelsList []string) generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_serviceaccount_labels", + "Kubernetes labels converted to Prometheus labels.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapServiceAccountFunc(func(sa *v1.ServiceAccount) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", sa.Labels, allowLabelsList) + m := metric.Metric{ + LabelKeys: labelKeys, + LabelValues: labelValues, + Value: 1, + } + return &metric.Family{ + Metrics: []*metric.Metric{&m}, + } + }), + ) +} + +func wrapServiceAccountFunc(f func(*v1.ServiceAccount) *metric.Family) func(interface{}) *metric.Family { + return func(obj interface{}) *metric.Family { + serviceAccount := obj.(*v1.ServiceAccount) + + metricFamily := f(serviceAccount) + + for _, m := range metricFamily.Metrics { + m.LabelKeys, m.LabelValues = mergeKeyValues(descServiceAccountLabelsDefaultLabels, []string{serviceAccount.Namespace, serviceAccount.Name, string(serviceAccount.UID)}, m.LabelKeys, m.LabelValues) + } + + return metricFamily + } +} + +func createServiceAccountListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { + return &cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + opts.FieldSelector = fieldSelector + return kubeClient.CoreV1().ServiceAccounts(ns).List(context.TODO(), opts) + }, + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + opts.FieldSelector = fieldSelector + return kubeClient.CoreV1().ServiceAccounts(ns).Watch(context.TODO(), opts) + }, + } +} diff --git a/internal/store/serviceaccount_test.go b/internal/store/serviceaccount_test.go new file mode 100644 index 0000000000..1223a5b6af --- /dev/null +++ b/internal/store/serviceaccount_test.go @@ -0,0 +1,88 @@ +/* +Copyright 2022 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "testing" + "time" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +func TestServiceAccountStore(t *testing.T) { + cases := []generateMetricsTestCase{ + { + Obj: &v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "serviceAccountName", + CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, + DeletionTimestamp: &metav1.Time{Time: time.Unix(3000000000, 0)}, + Namespace: "serviceAccountNS", + UID: "serviceAccountUID", + }, + AutomountServiceAccountToken: pointer.Bool(true), + Secrets: []v1.ObjectReference{ + { + APIVersion: "v1", + Kind: "Secret", + Name: "secretName", + Namespace: "serviceAccountNS", + }, + }, + ImagePullSecrets: []v1.LocalObjectReference{ + { + Name: "imagePullSecretName", + }, + }, + }, + Want: ` + # HELP kube_serviceaccount_info Information about a service account + # HELP kube_serviceaccount_created Unix creation timestamp + # HELP kube_serviceaccount_deleted Unix deletion timestamp + # HELP kube_serviceaccount_secret Secret being referenced by a service account + # HELP kube_serviceaccount_image_pull_secret Secret being referenced by a service account for the purpose of pulling images + # TYPE kube_serviceaccount_info gauge + # TYPE kube_serviceaccount_created gauge + # TYPE kube_serviceaccount_deleted gauge + # TYPE kube_serviceaccount_secret gauge + # TYPE kube_serviceaccount_image_pull_secret gauge + kube_serviceaccount_info{namespace="serviceAccountNS",serviceaccount="serviceAccountName",uid="serviceAccountUID",automount_token="true"} 1 + kube_serviceaccount_created{namespace="serviceAccountNS",serviceaccount="serviceAccountName",uid="serviceAccountUID"} 1.5e+09 + kube_serviceaccount_deleted{namespace="serviceAccountNS",serviceaccount="serviceAccountName",uid="serviceAccountUID"} 3e+09 + kube_serviceaccount_secret{namespace="serviceAccountNS",serviceaccount="serviceAccountName",uid="serviceAccountUID",name="secretName"} 1 + kube_serviceaccount_image_pull_secret{namespace="serviceAccountNS",serviceaccount="serviceAccountName",uid="serviceAccountUID",name="imagePullSecretName"} 1`, + MetricNames: []string{ + "kube_serviceaccount_info", + "kube_serviceaccount_created", + "kube_serviceaccount_deleted", + "kube_serviceaccount_secret", + "kube_serviceaccount_image_pull_secret", + }, + }, + } + for i, c := range cases { + c.Func = generator.ComposeMetricGenFuncs(serviceAccountMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) + c.Headers = generator.ExtractMetricFamilyHeaders(serviceAccountMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList)) + if err := c.run(); err != nil { + t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) + } + } +} diff --git a/internal/store/statefulset.go b/internal/store/statefulset.go index e7f6b0cbf8..d5d57e5bb3 100644 --- a/internal/store/statefulset.go +++ b/internal/store/statefulset.go @@ -17,7 +17,12 @@ limitations under the License. package store import ( - "k8s.io/kube-state-metrics/pkg/metric" + "context" + + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" v1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -28,16 +33,22 @@ import ( ) var ( + descStatefulSetAnnotationsName = "kube_statefulset_annotations" + descStatefulSetAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." descStatefulSetLabelsName = "kube_statefulset_labels" descStatefulSetLabelsHelp = "Kubernetes labels converted to Prometheus labels." descStatefulSetLabelsDefaultLabels = []string{"namespace", "statefulset"} +) - statefulSetMetricFamilies = []metric.FamilyGenerator{ - { - Name: "kube_statefulset_created", - Type: metric.Gauge, - Help: "Unix creation timestamp", - GenerateFunc: wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { +func statefulSetMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_statefulset_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.STABLE, + "", + wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { ms := []*metric.Metric{} if !s.CreationTimestamp.IsZero() { @@ -50,12 +61,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_statefulset_status_replicas", - Type: metric.Gauge, - Help: "The number of replicas per StatefulSet.", - GenerateFunc: wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_statefulset_status_replicas", + "The number of replicas per StatefulSet.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -64,12 +77,30 @@ var ( }, } }), - }, - { - Name: "kube_statefulset_status_replicas_current", - Type: metric.Gauge, - Help: "The number of current replicas per StatefulSet.", - GenerateFunc: wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_statefulset_status_replicas_available", + "The number of available replicas per StatefulSet.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { + return &metric.Family{ + Metrics: []*metric.Metric{ + { + Value: float64(s.Status.AvailableReplicas), + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_statefulset_status_replicas_current", + "The number of current replicas per StatefulSet.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -78,12 +109,14 @@ var ( }, } }), - }, - { - Name: "kube_statefulset_status_replicas_ready", - Type: metric.Gauge, - Help: "The number of ready replicas per StatefulSet.", - GenerateFunc: wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_statefulset_status_replicas_ready", + "The number of ready replicas per StatefulSet.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -92,12 +125,14 @@ var ( }, } }), - }, - { - Name: "kube_statefulset_status_replicas_updated", - Type: metric.Gauge, - Help: "The number of updated replicas per StatefulSet.", - GenerateFunc: wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_statefulset_status_replicas_updated", + "The number of updated replicas per StatefulSet.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -106,12 +141,14 @@ var ( }, } }), - }, - { - Name: "kube_statefulset_status_observed_generation", - Type: metric.Gauge, - Help: "The generation observed by the StatefulSet controller.", - GenerateFunc: wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_statefulset_status_observed_generation", + "The generation observed by the StatefulSet controller.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -120,12 +157,14 @@ var ( }, } }), - }, - { - Name: "kube_statefulset_replicas", - Type: metric.Gauge, - Help: "Number of desired pods for a StatefulSet.", - GenerateFunc: wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_statefulset_replicas", + "Number of desired pods for a StatefulSet.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { ms := []*metric.Metric{} if s.Spec.Replicas != nil { @@ -138,12 +177,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_statefulset_metadata_generation", - Type: metric.Gauge, - Help: "Sequence number representing a specific generation of the desired state for the StatefulSet.", - GenerateFunc: wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_statefulset_metadata_generation", + "Sequence number representing a specific generation of the desired state for the StatefulSet.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -152,13 +193,61 @@ var ( }, } }), - }, - { - Name: descStatefulSetLabelsName, - Type: metric.Gauge, - Help: descStatefulSetLabelsHelp, - GenerateFunc: wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(s.Labels) + ), + *generator.NewFamilyGeneratorWithStability( + "kube_statefulset_persistentvolumeclaim_retention_policy", + "Count of retention policy for StatefulSet template PVCs", + metric.Gauge, + basemetrics.ALPHA, + + "", + wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { + deletedPolicyLabel := "" + scaledPolicyLabel := "" + if policy := s.Spec.PersistentVolumeClaimRetentionPolicy; policy != nil { + deletedPolicyLabel = string(policy.WhenDeleted) + scaledPolicyLabel = string(policy.WhenScaled) + } + ms := []*metric.Metric{} + if deletedPolicyLabel != "" || scaledPolicyLabel != "" { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"when_deleted", "when_scaled"}, + LabelValues: []string{deletedPolicyLabel, scaledPolicyLabel}, + Value: 1, + }) + } + return &metric.Family{ + Metrics: ms, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descStatefulSetAnnotationsName, + descStatefulSetAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", s.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descStatefulSetLabelsName, + descStatefulSetLabelsHelp, + metric.Gauge, + basemetrics.STABLE, + "", + wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", s.Labels, allowLabelsList) return &metric.Family{ Metrics: []*metric.Metric{ { @@ -169,12 +258,14 @@ var ( }, } }), - }, - { - Name: "kube_statefulset_status_current_revision", - Type: metric.Gauge, - Help: "Indicates the version of the StatefulSet used to generate Pods in the sequence [0,currentReplicas).", - GenerateFunc: wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_statefulset_status_current_revision", + "Indicates the version of the StatefulSet used to generate Pods in the sequence [0,currentReplicas).", + metric.Gauge, + basemetrics.STABLE, + "", + wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -185,12 +276,14 @@ var ( }, } }), - }, - { - Name: "kube_statefulset_status_update_revision", - Type: metric.Gauge, - Help: "Indicates the version of the StatefulSet used to generate Pods in the sequence [replicas-updatedReplicas,replicas)", - GenerateFunc: wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_statefulset_status_update_revision", + "Indicates the version of the StatefulSet used to generate Pods in the sequence [replicas-updatedReplicas,replicas)", + metric.Gauge, + basemetrics.STABLE, + "", + wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -201,9 +294,9 @@ var ( }, } }), - }, + ), } -) +} func wrapStatefulSetFunc(f func(*v1.StatefulSet) *metric.Family) func(interface{}) *metric.Family { return func(obj interface{}) *metric.Family { @@ -212,21 +305,22 @@ func wrapStatefulSetFunc(f func(*v1.StatefulSet) *metric.Family) func(interface{ metricFamily := f(statefulSet) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descStatefulSetLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{statefulSet.Namespace, statefulSet.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descStatefulSetLabelsDefaultLabels, []string{statefulSet.Namespace, statefulSet.Name}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createStatefulSetListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createStatefulSetListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.AppsV1().StatefulSets(ns).List(opts) + opts.FieldSelector = fieldSelector + return kubeClient.AppsV1().StatefulSets(ns).List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.AppsV1().StatefulSets(ns).Watch(opts) + opts.FieldSelector = fieldSelector + return kubeClient.AppsV1().StatefulSets(ns).Watch(context.TODO(), opts) }, } } diff --git a/internal/store/statefulset_test.go b/internal/store/statefulset_test.go index b7162e871b..e48a96d961 100644 --- a/internal/store/statefulset_test.go +++ b/internal/store/statefulset_test.go @@ -23,7 +23,7 @@ import ( v1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) var ( @@ -60,24 +60,28 @@ func TestStatefulSetStore(t *testing.T) { }, }, Want: ` - # HELP kube_statefulset_created Unix creation timestamp - # HELP kube_statefulset_labels Kubernetes labels converted to Prometheus labels. - # HELP kube_statefulset_metadata_generation Sequence number representing a specific generation of the desired state for the StatefulSet. - # HELP kube_statefulset_replicas Number of desired pods for a StatefulSet. - # HELP kube_statefulset_status_current_revision Indicates the version of the StatefulSet used to generate Pods in the sequence [0,currentReplicas). - # HELP kube_statefulset_status_observed_generation The generation observed by the StatefulSet controller. - # HELP kube_statefulset_status_replicas The number of replicas per StatefulSet. - # HELP kube_statefulset_status_replicas_current The number of current replicas per StatefulSet. - # HELP kube_statefulset_status_replicas_ready The number of ready replicas per StatefulSet. - # HELP kube_statefulset_status_replicas_updated The number of updated replicas per StatefulSet. - # HELP kube_statefulset_status_update_revision Indicates the version of the StatefulSet used to generate Pods in the sequence [replicas-updatedReplicas,replicas) + # HELP kube_statefulset_created [STABLE] Unix creation timestamp + # HELP kube_statefulset_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # HELP kube_statefulset_metadata_generation [STABLE] Sequence number representing a specific generation of the desired state for the StatefulSet. + # HELP kube_statefulset_persistentvolumeclaim_retention_policy Count of retention policy for StatefulSet template PVCs + # HELP kube_statefulset_replicas [STABLE] Number of desired pods for a StatefulSet. + # HELP kube_statefulset_status_current_revision [STABLE] Indicates the version of the StatefulSet used to generate Pods in the sequence [0,currentReplicas). + # HELP kube_statefulset_status_observed_generation [STABLE] The generation observed by the StatefulSet controller. + # HELP kube_statefulset_status_replicas [STABLE] The number of replicas per StatefulSet. + # HELP kube_statefulset_status_replicas_available The number of available replicas per StatefulSet. + # HELP kube_statefulset_status_replicas_current [STABLE] The number of current replicas per StatefulSet. + # HELP kube_statefulset_status_replicas_ready [STABLE] The number of ready replicas per StatefulSet. + # HELP kube_statefulset_status_replicas_updated [STABLE] The number of updated replicas per StatefulSet. + # HELP kube_statefulset_status_update_revision [STABLE] Indicates the version of the StatefulSet used to generate Pods in the sequence [replicas-updatedReplicas,replicas) # TYPE kube_statefulset_created gauge # TYPE kube_statefulset_labels gauge # TYPE kube_statefulset_metadata_generation gauge + # TYPE kube_statefulset_persistentvolumeclaim_retention_policy gauge # TYPE kube_statefulset_replicas gauge # TYPE kube_statefulset_status_current_revision gauge # TYPE kube_statefulset_status_observed_generation gauge # TYPE kube_statefulset_status_replicas gauge + # TYPE kube_statefulset_status_replicas_available gauge # TYPE kube_statefulset_status_replicas_current gauge # TYPE kube_statefulset_status_replicas_ready gauge # TYPE kube_statefulset_status_replicas_updated gauge @@ -86,13 +90,14 @@ func TestStatefulSetStore(t *testing.T) { kube_statefulset_created{namespace="ns1",statefulset="statefulset1"} 1.5e+09 kube_statefulset_status_current_revision{namespace="ns1",revision="cr1",statefulset="statefulset1"} 1 kube_statefulset_status_replicas{namespace="ns1",statefulset="statefulset1"} 2 + kube_statefulset_status_replicas_available{namespace="ns1",statefulset="statefulset1"} 0 kube_statefulset_status_replicas_current{namespace="ns1",statefulset="statefulset1"} 0 kube_statefulset_status_replicas_ready{namespace="ns1",statefulset="statefulset1"} 0 kube_statefulset_status_replicas_updated{namespace="ns1",statefulset="statefulset1"} 0 kube_statefulset_status_observed_generation{namespace="ns1",statefulset="statefulset1"} 1 kube_statefulset_replicas{namespace="ns1",statefulset="statefulset1"} 3 kube_statefulset_metadata_generation{namespace="ns1",statefulset="statefulset1"} 3 - kube_statefulset_labels{label_app="example1",namespace="ns1",statefulset="statefulset1"} 1 + kube_statefulset_labels{namespace="ns1",statefulset="statefulset1"} 1 `, MetricNames: []string{ "kube_statefulset_created", @@ -101,11 +106,13 @@ func TestStatefulSetStore(t *testing.T) { "kube_statefulset_replicas", "kube_statefulset_status_observed_generation", "kube_statefulset_status_replicas", + "kube_statefulset_status_replicas_available", "kube_statefulset_status_replicas_current", "kube_statefulset_status_replicas_ready", "kube_statefulset_status_replicas_updated", "kube_statefulset_status_update_revision", "kube_statefulset_status_current_revision", + "kube_statefulset_persistentvolumeclaim_retention_policy", }, }, { @@ -127,41 +134,47 @@ func TestStatefulSetStore(t *testing.T) { ObservedGeneration: statefulSet2ObservedGeneration, ReadyReplicas: 5, Replicas: 5, + AvailableReplicas: 4, UpdatedReplicas: 3, UpdateRevision: "ur2", CurrentRevision: "cr2", }, }, Want: ` - # HELP kube_statefulset_labels Kubernetes labels converted to Prometheus labels. - # HELP kube_statefulset_metadata_generation Sequence number representing a specific generation of the desired state for the StatefulSet. - # HELP kube_statefulset_replicas Number of desired pods for a StatefulSet. - # HELP kube_statefulset_status_current_revision Indicates the version of the StatefulSet used to generate Pods in the sequence [0,currentReplicas). - # HELP kube_statefulset_status_observed_generation The generation observed by the StatefulSet controller. - # HELP kube_statefulset_status_replicas The number of replicas per StatefulSet. - # HELP kube_statefulset_status_replicas_current The number of current replicas per StatefulSet. - # HELP kube_statefulset_status_replicas_ready The number of ready replicas per StatefulSet. - # HELP kube_statefulset_status_replicas_updated The number of updated replicas per StatefulSet. - # HELP kube_statefulset_status_update_revision Indicates the version of the StatefulSet used to generate Pods in the sequence [replicas-updatedReplicas,replicas) + # HELP kube_statefulset_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # HELP kube_statefulset_metadata_generation [STABLE] Sequence number representing a specific generation of the desired state for the StatefulSet. + # HELP kube_statefulset_persistentvolumeclaim_retention_policy Count of retention policy for StatefulSet template PVCs + # HELP kube_statefulset_replicas [STABLE] Number of desired pods for a StatefulSet. + # HELP kube_statefulset_status_current_revision [STABLE] Indicates the version of the StatefulSet used to generate Pods in the sequence [0,currentReplicas). + # HELP kube_statefulset_status_observed_generation [STABLE] The generation observed by the StatefulSet controller. + # HELP kube_statefulset_status_replicas [STABLE] The number of replicas per StatefulSet. + # HELP kube_statefulset_status_replicas_available The number of available replicas per StatefulSet. + # HELP kube_statefulset_status_replicas_current [STABLE] The number of current replicas per StatefulSet. + # HELP kube_statefulset_status_replicas_ready [STABLE] The number of ready replicas per StatefulSet. + # HELP kube_statefulset_status_replicas_updated [STABLE] The number of updated replicas per StatefulSet. + # HELP kube_statefulset_status_update_revision [STABLE] Indicates the version of the StatefulSet used to generate Pods in the sequence [replicas-updatedReplicas,replicas) # TYPE kube_statefulset_labels gauge # TYPE kube_statefulset_metadata_generation gauge + # TYPE kube_statefulset_persistentvolumeclaim_retention_policy gauge # TYPE kube_statefulset_replicas gauge # TYPE kube_statefulset_status_current_revision gauge # TYPE kube_statefulset_status_observed_generation gauge # TYPE kube_statefulset_status_replicas gauge + # TYPE kube_statefulset_status_replicas_available gauge # TYPE kube_statefulset_status_replicas_current gauge # TYPE kube_statefulset_status_replicas_ready gauge # TYPE kube_statefulset_status_replicas_updated gauge # TYPE kube_statefulset_status_update_revision gauge kube_statefulset_status_update_revision{namespace="ns2",revision="ur2",statefulset="statefulset2"} 1 - kube_statefulset_status_replicas{namespace="ns2",statefulset="statefulset2"} 5 + kube_statefulset_status_replicas{namespace="ns2",statefulset="statefulset2"} 5 + kube_statefulset_status_replicas_available{namespace="ns2",statefulset="statefulset2"} 4 kube_statefulset_status_replicas_current{namespace="ns2",statefulset="statefulset2"} 2 kube_statefulset_status_replicas_ready{namespace="ns2",statefulset="statefulset2"} 5 kube_statefulset_status_replicas_updated{namespace="ns2",statefulset="statefulset2"} 3 - kube_statefulset_status_observed_generation{namespace="ns2",statefulset="statefulset2"} 2 - kube_statefulset_replicas{namespace="ns2",statefulset="statefulset2"} 6 - kube_statefulset_metadata_generation{namespace="ns2",statefulset="statefulset2"} 21 - kube_statefulset_labels{label_app="example2",namespace="ns2",statefulset="statefulset2"} 1 + kube_statefulset_status_observed_generation{namespace="ns2",statefulset="statefulset2"} 2 + kube_statefulset_replicas{namespace="ns2",statefulset="statefulset2"} 6 + kube_statefulset_metadata_generation{namespace="ns2",statefulset="statefulset2"} 21 + kube_statefulset_labels{namespace="ns2",statefulset="statefulset2"} 1 kube_statefulset_status_current_revision{namespace="ns2",revision="cr2",statefulset="statefulset2"} 1 `, MetricNames: []string{ @@ -170,11 +183,13 @@ func TestStatefulSetStore(t *testing.T) { "kube_statefulset_replicas", "kube_statefulset_status_observed_generation", "kube_statefulset_status_replicas", + "kube_statefulset_status_replicas_available", "kube_statefulset_status_replicas_current", "kube_statefulset_status_replicas_ready", "kube_statefulset_status_replicas_updated", "kube_statefulset_status_update_revision", "kube_statefulset_status_current_revision", + "kube_statefulset_persistentvolumeclaim_retention_policy", }, }, { @@ -199,32 +214,37 @@ func TestStatefulSetStore(t *testing.T) { }, }, Want: ` - # HELP kube_statefulset_labels Kubernetes labels converted to Prometheus labels. - # HELP kube_statefulset_metadata_generation Sequence number representing a specific generation of the desired state for the StatefulSet. - # HELP kube_statefulset_replicas Number of desired pods for a StatefulSet. - # HELP kube_statefulset_status_current_revision Indicates the version of the StatefulSet used to generate Pods in the sequence [0,currentReplicas). - # HELP kube_statefulset_status_replicas The number of replicas per StatefulSet. - # HELP kube_statefulset_status_replicas_current The number of current replicas per StatefulSet. - # HELP kube_statefulset_status_replicas_ready The number of ready replicas per StatefulSet. - # HELP kube_statefulset_status_replicas_updated The number of updated replicas per StatefulSet. - # HELP kube_statefulset_status_update_revision Indicates the version of the StatefulSet used to generate Pods in the sequence [replicas-updatedReplicas,replicas) + # HELP kube_statefulset_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # HELP kube_statefulset_metadata_generation [STABLE] Sequence number representing a specific generation of the desired state for the StatefulSet. + # HELP kube_statefulset_persistentvolumeclaim_retention_policy Count of retention policy for StatefulSet template PVCs + # HELP kube_statefulset_replicas [STABLE] Number of desired pods for a StatefulSet. + # HELP kube_statefulset_status_current_revision [STABLE] Indicates the version of the StatefulSet used to generate Pods in the sequence [0,currentReplicas). + # HELP kube_statefulset_status_replicas [STABLE] The number of replicas per StatefulSet. + # HELP kube_statefulset_status_replicas_available The number of available replicas per StatefulSet. + # HELP kube_statefulset_status_replicas_current [STABLE] The number of current replicas per StatefulSet. + # HELP kube_statefulset_status_replicas_ready [STABLE] The number of ready replicas per StatefulSet. + # HELP kube_statefulset_status_replicas_updated [STABLE] The number of updated replicas per StatefulSet. + # HELP kube_statefulset_status_update_revision [STABLE] Indicates the version of the StatefulSet used to generate Pods in the sequence [replicas-updatedReplicas,replicas) # TYPE kube_statefulset_labels gauge # TYPE kube_statefulset_metadata_generation gauge + # TYPE kube_statefulset_persistentvolumeclaim_retention_policy gauge # TYPE kube_statefulset_replicas gauge # TYPE kube_statefulset_status_current_revision gauge # TYPE kube_statefulset_status_replicas gauge + # TYPE kube_statefulset_status_replicas_available gauge # TYPE kube_statefulset_status_replicas_current gauge # TYPE kube_statefulset_status_replicas_ready gauge # TYPE kube_statefulset_status_replicas_updated gauge # TYPE kube_statefulset_status_update_revision gauge kube_statefulset_status_update_revision{namespace="ns3",revision="ur3",statefulset="statefulset3"} 1 - kube_statefulset_status_replicas{namespace="ns3",statefulset="statefulset3"} 7 + kube_statefulset_status_replicas{namespace="ns3",statefulset="statefulset3"} 7 + kube_statefulset_status_replicas_available{namespace="ns3",statefulset="statefulset3"} 0 kube_statefulset_status_replicas_current{namespace="ns3",statefulset="statefulset3"} 0 kube_statefulset_status_replicas_ready{namespace="ns3",statefulset="statefulset3"} 0 kube_statefulset_status_replicas_updated{namespace="ns3",statefulset="statefulset3"} 0 - kube_statefulset_replicas{namespace="ns3",statefulset="statefulset3"} 9 - kube_statefulset_metadata_generation{namespace="ns3",statefulset="statefulset3"} 36 - kube_statefulset_labels{label_app="example3",namespace="ns3",statefulset="statefulset3"} 1 + kube_statefulset_replicas{namespace="ns3",statefulset="statefulset3"} 9 + kube_statefulset_metadata_generation{namespace="ns3",statefulset="statefulset3"} 36 + kube_statefulset_labels{namespace="ns3",statefulset="statefulset3"} 1 kube_statefulset_status_current_revision{namespace="ns3",revision="cr3",statefulset="statefulset3"} 1 `, MetricNames: []string{ @@ -232,19 +252,95 @@ func TestStatefulSetStore(t *testing.T) { "kube_statefulset_metadata_generation", "kube_statefulset_replicas", "kube_statefulset_status_replicas", + "kube_statefulset_status_replicas_available", "kube_statefulset_status_replicas_current", "kube_statefulset_status_replicas_ready", "kube_statefulset_status_replicas_updated", "kube_statefulset_status_update_revision", "kube_statefulset_status_current_revision", + "kube_statefulset_persistentvolumeclaim_retention_policy", + }, + }, + { + Obj: &v1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "statefulset4", + Namespace: "ns4", + Labels: map[string]string{ + "app": "example4", + }, + Generation: 1, + }, + Spec: v1.StatefulSetSpec{ + Replicas: &statefulSet1Replicas, + ServiceName: "statefulset4service", + PersistentVolumeClaimRetentionPolicy: &v1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: v1.RetainPersistentVolumeClaimRetentionPolicyType, + WhenScaled: v1.DeletePersistentVolumeClaimRetentionPolicyType, + }, + }, + Status: v1.StatefulSetStatus{ + ObservedGeneration: 0, + Replicas: 7, + UpdateRevision: "ur3", + CurrentRevision: "cr3", + }, + }, + Want: ` + # HELP kube_statefulset_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # HELP kube_statefulset_metadata_generation [STABLE] Sequence number representing a specific generation of the desired state for the StatefulSet. + # HELP kube_statefulset_persistentvolumeclaim_retention_policy Count of retention policy for StatefulSet template PVCs + # HELP kube_statefulset_replicas [STABLE] Number of desired pods for a StatefulSet. + # HELP kube_statefulset_status_current_revision [STABLE] Indicates the version of the StatefulSet used to generate Pods in the sequence [0,currentReplicas). + # HELP kube_statefulset_status_replicas [STABLE] The number of replicas per StatefulSet. + # HELP kube_statefulset_status_replicas_available The number of available replicas per StatefulSet. + # HELP kube_statefulset_status_replicas_current [STABLE] The number of current replicas per StatefulSet. + # HELP kube_statefulset_status_replicas_ready [STABLE] The number of ready replicas per StatefulSet. + # HELP kube_statefulset_status_replicas_updated [STABLE] The number of updated replicas per StatefulSet. + # HELP kube_statefulset_status_update_revision [STABLE] Indicates the version of the StatefulSet used to generate Pods in the sequence [replicas-updatedReplicas,replicas) + # TYPE kube_statefulset_labels gauge + # TYPE kube_statefulset_metadata_generation gauge + # TYPE kube_statefulset_persistentvolumeclaim_retention_policy gauge + # TYPE kube_statefulset_replicas gauge + # TYPE kube_statefulset_status_current_revision gauge + # TYPE kube_statefulset_status_replicas gauge + # TYPE kube_statefulset_status_replicas_available gauge + # TYPE kube_statefulset_status_replicas_current gauge + # TYPE kube_statefulset_status_replicas_ready gauge + # TYPE kube_statefulset_status_replicas_updated gauge + # TYPE kube_statefulset_status_update_revision gauge + kube_statefulset_status_update_revision{namespace="ns4",revision="ur3",statefulset="statefulset4"} 1 + kube_statefulset_status_replicas{namespace="ns4",statefulset="statefulset4"} 7 + kube_statefulset_status_replicas_available{namespace="ns4",statefulset="statefulset4"} 0 + kube_statefulset_status_replicas_current{namespace="ns4",statefulset="statefulset4"} 0 + kube_statefulset_status_replicas_ready{namespace="ns4",statefulset="statefulset4"} 0 + kube_statefulset_status_replicas_updated{namespace="ns4",statefulset="statefulset4"} 0 + kube_statefulset_replicas{namespace="ns4",statefulset="statefulset4"} 3 + kube_statefulset_metadata_generation{namespace="ns4",statefulset="statefulset4"} 1 + kube_statefulset_persistentvolumeclaim_retention_policy{namespace="ns4",statefulset="statefulset4",when_deleted="Retain",when_scaled="Delete"} 1 + kube_statefulset_labels{namespace="ns4",statefulset="statefulset4"} 1 + kube_statefulset_status_current_revision{namespace="ns4",revision="cr3",statefulset="statefulset4"} 1 + `, + MetricNames: []string{ + "kube_statefulset_labels", + "kube_statefulset_metadata_generation", + "kube_statefulset_replicas", + "kube_statefulset_status_replicas", + "kube_statefulset_status_replicas_available", + "kube_statefulset_status_replicas_current", + "kube_statefulset_status_replicas_ready", + "kube_statefulset_status_replicas_updated", + "kube_statefulset_status_update_revision", + "kube_statefulset_status_current_revision", + "kube_statefulset_persistentvolumeclaim_retention_policy", }, }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(statefulSetMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(statefulSetMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(statefulSetMetricFamilies(nil, nil)) + c.Headers = generator.ExtractMetricFamilyHeaders(statefulSetMetricFamilies(nil, nil)) if err := c.run(); err != nil { - t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) + t.Errorf("unexpected collecting result for statefulset%d run:\n%s", i+1, err) } } } diff --git a/internal/store/storageclass.go b/internal/store/storageclass.go index 22cbe9b7dc..f088953f6a 100644 --- a/internal/store/storageclass.go +++ b/internal/store/storageclass.go @@ -14,7 +14,12 @@ limitations under the License. package store import ( - "k8s.io/kube-state-metrics/pkg/metric" + "context" + + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" @@ -26,18 +31,24 @@ import ( ) var ( + descStorageClassAnnotationsName = "kube_storageclass_annotations" + descStorageClassAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." descStorageClassLabelsName = "kube_storageclass_labels" descStorageClassLabelsHelp = "Kubernetes labels converted to Prometheus labels." descStorageClassLabelsDefaultLabels = []string{"storageclass"} defaultReclaimPolicy = v1.PersistentVolumeReclaimDelete defaultVolumeBindingMode = storagev1.VolumeBindingImmediate +) - storageClassMetricFamilies = []metric.FamilyGenerator{ - { - Name: "kube_storageclass_info", - Type: metric.Gauge, - Help: "Information about storageclass.", - GenerateFunc: wrapStorageClassFunc(func(s *storagev1.StorageClass) *metric.Family { +func storageClassMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_storageclass_info", + "Information about storageclass.", + metric.Gauge, + basemetrics.STABLE, + "", + wrapStorageClassFunc(func(s *storagev1.StorageClass) *metric.Family { // Add default values if missing. if s.ReclaimPolicy == nil { @@ -49,18 +60,20 @@ var ( } m := metric.Metric{ - LabelKeys: []string{"provisioner", "reclaimPolicy", "volumeBindingMode"}, + LabelKeys: []string{"provisioner", "reclaim_policy", "volume_binding_mode"}, LabelValues: []string{s.Provisioner, string(*s.ReclaimPolicy), string(*s.VolumeBindingMode)}, Value: 1, } return &metric.Family{Metrics: []*metric.Metric{&m}} }), - }, - { - Name: "kube_storageclass_created", - Type: metric.Gauge, - Help: "Unix creation timestamp", - GenerateFunc: wrapStorageClassFunc(func(s *storagev1.StorageClass) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_storageclass_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.STABLE, + "", + wrapStorageClassFunc(func(s *storagev1.StorageClass) *metric.Family { ms := []*metric.Metric{} if !s.CreationTimestamp.IsZero() { ms = append(ms, &metric.Metric{ @@ -71,13 +84,34 @@ var ( Metrics: ms, } }), - }, - { - Name: descStorageClassLabelsName, - Type: metric.Gauge, - Help: descStorageClassLabelsHelp, - GenerateFunc: wrapStorageClassFunc(func(s *storagev1.StorageClass) *metric.Family { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(s.Labels) + ), + *generator.NewFamilyGeneratorWithStability( + descStorageClassAnnotationsName, + descStorageClassAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapStorageClassFunc(func(s *storagev1.StorageClass) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", s.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descStorageClassLabelsName, + descStorageClassLabelsHelp, + metric.Gauge, + basemetrics.STABLE, + "", + wrapStorageClassFunc(func(s *storagev1.StorageClass) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", s.Labels, allowLabelsList) return &metric.Family{ Metrics: []*metric.Metric{ { @@ -88,9 +122,9 @@ var ( }, } }), - }, + ), } -) +} func wrapStorageClassFunc(f func(*storagev1.StorageClass) *metric.Family) func(interface{}) *metric.Family { return func(obj interface{}) *metric.Family { @@ -99,21 +133,20 @@ func wrapStorageClassFunc(f func(*storagev1.StorageClass) *metric.Family) func(i metricFamily := f(storageClass) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descStorageClassLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{storageClass.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descStorageClassLabelsDefaultLabels, []string{storageClass.Name}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createStorageClassListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createStorageClassListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.StorageV1().StorageClasses().List(opts) + return kubeClient.StorageV1().StorageClasses().List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.StorageV1().StorageClasses().Watch(opts) + return kubeClient.StorageV1().StorageClasses().Watch(context.TODO(), opts) }, } } diff --git a/internal/store/storageclass_test.go b/internal/store/storageclass_test.go index bd826666eb..4f086f4826 100644 --- a/internal/store/storageclass_test.go +++ b/internal/store/storageclass_test.go @@ -20,7 +20,7 @@ import ( storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) func TestStorageClassStore(t *testing.T) { @@ -40,9 +40,9 @@ func TestStorageClassStore(t *testing.T) { VolumeBindingMode: &volumeBindingMode, }, Want: ` - # HELP kube_storageclass_info Information about storageclass. + # HELP kube_storageclass_info [STABLE] Information about storageclass. # TYPE kube_storageclass_info gauge - kube_storageclass_info{storageclass="test_storageclass-info",provisioner="kubernetes.io/rbd",reclaimPolicy="Delete",volumeBindingMode="Immediate"} 1 + kube_storageclass_info{storageclass="test_storageclass-info",provisioner="kubernetes.io/rbd",reclaim_policy="Delete",volume_binding_mode="Immediate"} 1 `, MetricNames: []string{ "kube_storageclass_info", @@ -58,9 +58,9 @@ func TestStorageClassStore(t *testing.T) { VolumeBindingMode: nil, }, Want: ` - # HELP kube_storageclass_info Information about storageclass. + # HELP kube_storageclass_info [STABLE] Information about storageclass. # TYPE kube_storageclass_info gauge - kube_storageclass_info{storageclass="test_storageclass-default-info",provisioner="kubernetes.io/rbd",reclaimPolicy="Delete",volumeBindingMode="Immediate"} 1 + kube_storageclass_info{storageclass="test_storageclass-default-info",provisioner="kubernetes.io/rbd",reclaim_policy="Delete",volume_binding_mode="Immediate"} 1 `, MetricNames: []string{ "kube_storageclass_info", @@ -77,7 +77,7 @@ func TestStorageClassStore(t *testing.T) { VolumeBindingMode: &volumeBindingMode, }, Want: ` - # HELP kube_storageclass_created Unix creation timestamp + # HELP kube_storageclass_created [STABLE] Unix creation timestamp # TYPE kube_storageclass_created gauge kube_storageclass_created{storageclass="test_kube_storageclass-created"} 1.501569018e+09 `, @@ -98,9 +98,9 @@ func TestStorageClassStore(t *testing.T) { VolumeBindingMode: &volumeBindingMode, }, Want: ` - # HELP kube_storageclass_labels Kubernetes labels converted to Prometheus labels. + # HELP kube_storageclass_labels [STABLE] Kubernetes labels converted to Prometheus labels. # TYPE kube_storageclass_labels gauge - kube_storageclass_labels{storageclass="test_storageclass-labels",label_foo="bar"} 1 + kube_storageclass_labels{storageclass="test_storageclass-labels"} 1 `, MetricNames: []string{ "kube_storageclass_labels", @@ -108,8 +108,8 @@ func TestStorageClassStore(t *testing.T) { }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(storageClassMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(storageClassMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(storageClassMetricFamilies(nil, nil)) + c.Headers = generator.ExtractMetricFamilyHeaders(storageClassMetricFamilies(nil, nil)) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/testutils.go b/internal/store/testutils.go index 58fb9cc2af..8de6c52ae2 100644 --- a/internal/store/testutils.go +++ b/internal/store/testutils.go @@ -24,17 +24,19 @@ import ( "sort" "strings" - "github.com/pkg/errors" + "github.com/google/go-cmp/cmp" - metricsstore "k8s.io/kube-state-metrics/pkg/metrics_store" + "k8s.io/kube-state-metrics/v2/pkg/metric" ) type generateMetricsTestCase struct { - Obj interface{} - MetricNames []string - Want string - Headers []string - Func func(interface{}) []metricsstore.FamilyByteSlicer + Obj interface{} + MetricNames []string + AllowAnnotationsList []string + AllowLabelsList []string + Want string + Headers []string + Func func(interface{}) []metric.FamilyInterface } func (testCase *generateMetricsTestCase) run() error { @@ -51,7 +53,7 @@ func (testCase *generateMetricsTestCase) run() error { out := headers + "\n" + metrics if err := compareOutput(testCase.Want, out); err != nil { - return errors.Wrap(err, "expected wanted output to equal output") + return fmt.Errorf("expected wanted output to equal output: %w", err) } return nil @@ -66,8 +68,8 @@ func compareOutput(expected, actual string) error { } } - if entities[0] != entities[1] { - return errors.Errorf("\nEXPECTED:\n--------------\n%v\nACTUAL:\n--------------\n%v", entities[0], entities[1]) + if diff := cmp.Diff(entities[0], entities[1]); diff != "" { + return fmt.Errorf("(-want, +got):\n%s", diff) } return nil diff --git a/internal/store/utils.go b/internal/store/utils.go index e6b66cd8b8..009d5739d7 100644 --- a/internal/store/utils.go +++ b/internal/store/utils.go @@ -23,15 +23,16 @@ import ( "strconv" "strings" - "k8s.io/apimachinery/pkg/util/validation" - v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/validation" - "k8s.io/kube-state-metrics/pkg/metric" + "k8s.io/kube-state-metrics/v2/pkg/metric" + "k8s.io/kube-state-metrics/v2/pkg/options" ) var ( invalidLabelCharRE = regexp.MustCompile(`[^a-zA-Z0-9_]`) + matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])") conditionStatuses = []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionFalse, v1.ConditionUnknown} ) @@ -72,29 +73,76 @@ func addConditionMetrics(cs v1.ConditionStatus) []*metric.Metric { return ms } -func kubeLabelsToPrometheusLabels(labels map[string]string) ([]string, []string) { - return mapToPrometheusLabels(labels, "label") +func kubeMapToPrometheusLabels(prefix string, input map[string]string) ([]string, []string) { + return mapToPrometheusLabels(input, prefix) } func mapToPrometheusLabels(labels map[string]string, prefix string) ([]string, []string) { labelKeys := make([]string, 0, len(labels)) - for k := range labels { - labelKeys = append(labelKeys, k) + labelValues := make([]string, 0, len(labels)) + + sortedKeys := make([]string, 0) + for key := range labels { + sortedKeys = append(sortedKeys, key) } - sort.Strings(labelKeys) + sort.Strings(sortedKeys) - labelValues := make([]string, 0, len(labels)) - for i, k := range labelKeys { - labelKeys[i] = prefix + "_" + sanitizeLabelName(k) + // conflictDesc holds some metadata for resolving potential label conflicts + type conflictDesc struct { + // the number of conflicting label keys we saw so far + count int + + // the offset of the initial conflicting label key, so we could + // later go back and rename "label_foo" to "label_foo_conflict1" + initial int + } + + conflicts := make(map[string]*conflictDesc) + for _, k := range sortedKeys { + labelKey := labelName(prefix, k) + if conflict, ok := conflicts[labelKey]; ok { + if conflict.count == 1 { + // this is the first conflict for the label, + // so we have to go back and rename the initial label that we've already added + labelKeys[conflict.initial] = labelConflictSuffix(labelKeys[conflict.initial], conflict.count) + } + + conflict.count++ + labelKey = labelConflictSuffix(labelKey, conflict.count) + } else { + // we'll need this info later in case there are conflicts + conflicts[labelKey] = &conflictDesc{ + count: 1, + initial: len(labelKeys), + } + } + labelKeys = append(labelKeys, labelKey) labelValues = append(labelValues, labels[k]) } return labelKeys, labelValues } +func labelName(prefix, labelName string) string { + return prefix + "_" + lintLabelName(sanitizeLabelName(labelName)) +} + func sanitizeLabelName(s string) string { return invalidLabelCharRE.ReplaceAllString(s, "_") } +func lintLabelName(s string) string { + return toSnakeCase(s) +} + +func toSnakeCase(s string) string { + snake := matchAllCap.ReplaceAllString(s, "${1}_${2}") + return strings.ToLower(snake) +} + +func labelConflictSuffix(label string, count int) string { + return fmt.Sprintf("%s_conflict%d", label, count) +} + func isHugePageResourceName(name v1.ResourceName) bool { return strings.HasPrefix(string(name), v1.ResourceHugePagesPrefix) } @@ -123,3 +171,46 @@ func isNativeResource(name v1.ResourceName) bool { func isPrefixedNativeResource(name v1.ResourceName) bool { return strings.Contains(string(name), v1.ResourceDefaultNamespacePrefix) } + +// createPrometheusLabelKeysValues takes in passed kubernetes annotations/labels +// and associated allowed list in kubernetes label format. +// It returns only those allowed annotations/labels that exist in the list and converts them to Prometheus labels. +func createPrometheusLabelKeysValues(prefix string, allKubeData map[string]string, allowList []string) ([]string, []string) { + allowedKubeData := make(map[string]string) + + if len(allowList) > 0 { + if allowList[0] == options.LabelWildcard { + return kubeMapToPrometheusLabels(prefix, allKubeData) + } + + for _, l := range allowList { + v, found := allKubeData[l] + if found { + allowedKubeData[l] = v + } + } + } + return kubeMapToPrometheusLabels(prefix, allowedKubeData) +} + +// mergeKeyValues merges label keys and values slice pairs into a single slice pair. +// Arguments are passed as equal-length pairs of slices, where the first slice contains keys and second contains values. +// Example: mergeKeyValues(keys1, values1, keys2, values2) => (keys1+keys2, values1+values2) +func mergeKeyValues(keyValues ...[]string) (keys, values []string) { + capacity := 0 + for i := 0; i < len(keyValues); i += 2 { + capacity += len(keyValues[i]) + } + + // Allocate one contiguous block, then split it up to keys and values zero'd slices. + keysValues := make([]string, 0, capacity*2) + keys = (keysValues[0:capacity:capacity])[:0] + values = (keysValues[capacity : capacity*2])[:0] + + for i := 0; i < len(keyValues); i += 2 { + keys = append(keys, keyValues[i]...) + values = append(values, keyValues[i+1]...) + } + + return keys, values +} diff --git a/internal/store/utils_test.go b/internal/store/utils_test.go index b7c6aebaf8..c91256e711 100644 --- a/internal/store/utils_test.go +++ b/internal/store/utils_test.go @@ -18,6 +18,7 @@ package store import ( "fmt" + "reflect" "testing" v1 "k8s.io/api/core/v1" @@ -176,11 +177,77 @@ func TestKubeLabelsToPrometheusLabels(t *testing.T) { expectKeys: []string{"label_an", "label_order", "label_test"}, expectValues: []string{"", "", ""}, }, + { + kubeLabels: map[string]string{ + "conflicting_label1": "underscore", + "conflicting.label1": "dot", + "conflicting-label1": "hyphen", + + "conflicting.label2": "dot", + "conflicting-label2": "hyphen", + "conflicting_label2": "underscore", + + "conflicting-label3": "hyphen", + "conflicting_label3": "underscore", + "conflicting.label3": "dot", + }, + // keys are sorted alphabetically during sanitization + expectKeys: []string{ + "label_conflicting_label1_conflict1", + "label_conflicting_label2_conflict1", + "label_conflicting_label3_conflict1", + "label_conflicting_label1_conflict2", + "label_conflicting_label2_conflict2", + "label_conflicting_label3_conflict2", + "label_conflicting_label1_conflict3", + "label_conflicting_label2_conflict3", + "label_conflicting_label3_conflict3", + }, + expectValues: []string{ + "hyphen", + "hyphen", + "hyphen", + "dot", + "dot", + "dot", + "underscore", + "underscore", + "underscore", + }, + }, + { + kubeLabels: map[string]string{ + "camelCase": "camel_case", + }, + expectKeys: []string{"label_camel_case"}, + expectValues: []string{"camel_case"}, + }, + { + kubeLabels: map[string]string{ + "snake_camelCase": "snake_and_camel_case", + }, + expectKeys: []string{"label_snake_camel_case"}, + expectValues: []string{"snake_and_camel_case"}, + }, + { + kubeLabels: map[string]string{ + "conflicting_camelCase": "camel_case", + "conflicting_camel_case": "snake_case", + }, + expectKeys: []string{ + "label_conflicting_camel_case_conflict1", + "label_conflicting_camel_case_conflict2", + }, + expectValues: []string{ + "camel_case", + "snake_case", + }, + }, } for _, tc := range testCases { t.Run(fmt.Sprintf("kubelabels input=%v , expected prometheus keys=%v, expected prometheus values=%v", tc.kubeLabels, tc.expectKeys, tc.expectValues), func(t *testing.T) { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(tc.kubeLabels) + labelKeys, labelValues := kubeMapToPrometheusLabels("label", tc.kubeLabels) if len(labelKeys) != len(tc.expectKeys) { t.Errorf("Got Prometheus label keys with len %d but expected %d", len(labelKeys), len(tc.expectKeys)) } @@ -198,3 +265,57 @@ func TestKubeLabelsToPrometheusLabels(t *testing.T) { } } + +func TestMergeKeyValues(t *testing.T) { + testCases := []struct { + name string + keyValuePairSlices [][]string + expectKeys []string + expectValues []string + }{ + { + name: "singlePair", + keyValuePairSlices: [][]string{ + {"keyA", "keyB", "keyC"}, + {"valueA", "valueB", "valueC"}, + }, + expectKeys: []string{"keyA", "keyB", "keyC"}, + expectValues: []string{"valueA", "valueB", "valueC"}, + }, + { + name: "evenPair", + keyValuePairSlices: [][]string{ + {"keyA", "keyB", "keyC"}, + {"valueA", "valueB", "valueC"}, + {"keyX", "keyY", "keyZ"}, + {"valueX", "valueY", "valueZ"}, + }, + expectKeys: []string{"keyA", "keyB", "keyC", "keyX", "keyY", "keyZ"}, + expectValues: []string{"valueA", "valueB", "valueC", "valueX", "valueY", "valueZ"}, + }, + { + name: "oddPair", + keyValuePairSlices: [][]string{ + {"keyA", "keyB", "keyC"}, + {"valueA", "valueB", "valueC"}, + {"keyX", "keyY", "keyZ"}, + {"valueX", "valueY", "valueZ"}, + {"keyM", "keyN", "keyP"}, + {"valueM", "valueN", "valueP"}, + }, + expectKeys: []string{"keyA", "keyB", "keyC", "keyX", "keyY", "keyZ", "keyM", "keyN", "keyP"}, + expectValues: []string{"valueA", "valueB", "valueC", "valueX", "valueY", "valueZ", "valueM", "valueN", "valueP"}, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gotKeys, gotValues := mergeKeyValues(tc.keyValuePairSlices...) + if !reflect.DeepEqual(gotKeys, tc.expectKeys) { + t.Errorf("mergeKeyValues() got = %v, want %v", gotKeys, tc.expectKeys) + } + if !reflect.DeepEqual(gotValues, tc.expectValues) { + t.Errorf("mergeKeyValues() got1 = %v, want %v", gotValues, tc.expectValues) + } + }) + } +} diff --git a/internal/store/validatingwebhookconfiguration.go b/internal/store/validatingwebhookconfiguration.go index 7742332c1f..da698acdd0 100644 --- a/internal/store/validatingwebhookconfiguration.go +++ b/internal/store/validatingwebhookconfiguration.go @@ -17,26 +17,31 @@ limitations under the License. package store import ( + "context" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" - "k8s.io/kube-state-metrics/pkg/metric" + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) var ( - descValidatingWebhookConfigurationHelp = "Kubernetes labels converted to Prometheus labels." descValidatingWebhookConfigurationDefaultLabels = []string{"namespace", "validatingwebhookconfiguration"} - validatingWebhookConfigurationMetricFamilies = []metric.FamilyGenerator{ - { - Name: "kube_validatingwebhookconfiguration_info", - Type: metric.Gauge, - Help: "Information about the ValidatingWebhookConfiguration.", - GenerateFunc: wrapValidatingWebhookConfigurationFunc(func(vwc *admissionregistrationv1.ValidatingWebhookConfiguration) *metric.Family { + validatingWebhookConfigurationMetricFamilies = []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_validatingwebhookconfiguration_info", + "Information about the ValidatingWebhookConfiguration.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapValidatingWebhookConfigurationFunc(func(vwc *admissionregistrationv1.ValidatingWebhookConfiguration) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -45,12 +50,14 @@ var ( }, } }), - }, - { - Name: "kube_validatingwebhookconfiguration_created", - Type: metric.Gauge, - Help: "Unix creation timestamp.", - GenerateFunc: wrapValidatingWebhookConfigurationFunc(func(vwc *admissionregistrationv1.ValidatingWebhookConfiguration) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_validatingwebhookconfiguration_created", + "Unix creation timestamp.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapValidatingWebhookConfigurationFunc(func(vwc *admissionregistrationv1.ValidatingWebhookConfiguration) *metric.Family { ms := []*metric.Metric{} if !vwc.CreationTimestamp.IsZero() { @@ -62,27 +69,29 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_validatingwebhookconfiguration_metadata_resource_version", - Type: metric.Gauge, - Help: "Resource version representing a specific version of the ValidatingWebhookConfiguration.", - GenerateFunc: wrapValidatingWebhookConfigurationFunc(func(vwc *admissionregistrationv1.ValidatingWebhookConfiguration) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_validatingwebhookconfiguration_metadata_resource_version", + "Resource version representing a specific version of the ValidatingWebhookConfiguration.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapValidatingWebhookConfigurationFunc(func(vwc *admissionregistrationv1.ValidatingWebhookConfiguration) *metric.Family { return &metric.Family{ Metrics: resourceVersionMetric(vwc.ObjectMeta.ResourceVersion), } }), - }, + ), } ) -func createValidatingWebhookConfigurationListWatch(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createValidatingWebhookConfigurationListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().List(opts) + return kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Watch(opts) + return kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Watch(context.TODO(), opts) }, } } @@ -94,8 +103,7 @@ func wrapValidatingWebhookConfigurationFunc(f func(*admissionregistrationv1.Vali metricFamily := f(mutatingWebhookConfiguration) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descValidatingWebhookConfigurationDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{mutatingWebhookConfiguration.Namespace, mutatingWebhookConfiguration.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descValidatingWebhookConfigurationDefaultLabels, []string{mutatingWebhookConfiguration.Namespace, mutatingWebhookConfiguration.Name}, m.LabelKeys, m.LabelValues) } return metricFamily diff --git a/internal/store/validatingwebhookconfiguration_test.go b/internal/store/validatingwebhookconfiguration_test.go index e0ba7d3f51..fc64eb2e4b 100644 --- a/internal/store/validatingwebhookconfiguration_test.go +++ b/internal/store/validatingwebhookconfiguration_test.go @@ -22,7 +22,7 @@ import ( admissionregistrationv1 "k8s.io/api/admissionregistration/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) func TestValidatingWebhookConfigurationStore(t *testing.T) { @@ -71,8 +71,8 @@ func TestValidatingWebhookConfigurationStore(t *testing.T) { }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(validatingWebhookConfigurationMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(validatingWebhookConfigurationMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(validatingWebhookConfigurationMetricFamilies) + c.Headers = generator.ExtractMetricFamilyHeaders(validatingWebhookConfigurationMetricFamilies) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/verticalpodautoscaler.go b/internal/store/verticalpodautoscaler.go index a9611ec3af..6d326049f8 100644 --- a/internal/store/verticalpodautoscaler.go +++ b/internal/store/verticalpodautoscaler.go @@ -17,6 +17,9 @@ limitations under the License. package store import ( + "context" + + autoscalingv1 "k8s.io/api/autoscaling/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -25,23 +28,50 @@ import ( vpaclientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" - "k8s.io/kube-state-metrics/pkg/constant" - "k8s.io/kube-state-metrics/pkg/metric" + "k8s.io/kube-state-metrics/v2/pkg/constant" + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) var ( + descVerticalPodAutoscalerAnnotationsName = "kube_verticalpodautoscaler_annotations" + descVerticalPodAutoscalerAnnotationsHelp = "Kubernetes annotations converted to Prometheus labels." descVerticalPodAutoscalerLabelsName = "kube_verticalpodautoscaler_labels" descVerticalPodAutoscalerLabelsHelp = "Kubernetes labels converted to Prometheus labels." descVerticalPodAutoscalerLabelsDefaultLabels = []string{"namespace", "verticalpodautoscaler", "target_api_version", "target_kind", "target_name"} +) - vpaMetricFamilies = []metric.FamilyGenerator{ - { - Name: descVerticalPodAutoscalerLabelsName, - Type: metric.Gauge, - Help: descVerticalPodAutoscalerLabelsHelp, - GenerateFunc: wrapVPAFunc(func(a *autoscaling.VerticalPodAutoscaler) *metric.Family { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(a.Labels) +func vpaMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + descVerticalPodAutoscalerAnnotationsName, + descVerticalPodAutoscalerAnnotationsHelp, + metric.Gauge, + basemetrics.ALPHA, + "v2.9.0", + wrapVPAFunc(func(a *autoscaling.VerticalPodAutoscaler) *metric.Family { + annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", a.Annotations, allowAnnotationsList) + return &metric.Family{ + Metrics: []*metric.Metric{ + { + LabelKeys: annotationKeys, + LabelValues: annotationValues, + Value: 1, + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + descVerticalPodAutoscalerLabelsName, + descVerticalPodAutoscalerLabelsHelp, + metric.Gauge, + basemetrics.ALPHA, + "v2.9.0", + wrapVPAFunc(func(a *autoscaling.VerticalPodAutoscaler) *metric.Family { + labelKeys, labelValues := createPrometheusLabelKeysValues("label", a.Labels, allowLabelsList) return &metric.Family{ Metrics: []*metric.Metric{ { @@ -52,12 +82,14 @@ var ( }, } }), - }, - { - Name: "kube_verticalpodautoscaler_spec_updatepolicy_updatemode", - Type: metric.Gauge, - Help: "Update mode of the VerticalPodAutoscaler.", - GenerateFunc: wrapVPAFunc(func(a *autoscaling.VerticalPodAutoscaler) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_verticalpodautoscaler_spec_updatepolicy_updatemode", + "Update mode of the VerticalPodAutoscaler.", + metric.Gauge, + basemetrics.ALPHA, + "v2.9.0", + wrapVPAFunc(func(a *autoscaling.VerticalPodAutoscaler) *metric.Family { ms := []*metric.Metric{} if a.Spec.UpdatePolicy == nil || a.Spec.UpdatePolicy.UpdateMode == nil { @@ -89,12 +121,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed", - Type: metric.Gauge, - Help: "Minimum resources the VerticalPodAutoscaler can set for containers matching the name.", - GenerateFunc: wrapVPAFunc(func(a *autoscaling.VerticalPodAutoscaler) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed", + "Minimum resources the VerticalPodAutoscaler can set for containers matching the name.", + metric.Gauge, + basemetrics.ALPHA, + "v2.9.0", + wrapVPAFunc(func(a *autoscaling.VerticalPodAutoscaler) *metric.Family { ms := []*metric.Metric{} if a.Spec.ResourcePolicy == nil || a.Spec.ResourcePolicy.ContainerPolicies == nil { return &metric.Family{ @@ -110,12 +144,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed", - Type: metric.Gauge, - Help: "Maximum resources the VerticalPodAutoscaler can set for containers matching the name.", - GenerateFunc: wrapVPAFunc(func(a *autoscaling.VerticalPodAutoscaler) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed", + "Maximum resources the VerticalPodAutoscaler can set for containers matching the name.", + metric.Gauge, + basemetrics.ALPHA, + "v2.9.0", + wrapVPAFunc(func(a *autoscaling.VerticalPodAutoscaler) *metric.Family { ms := []*metric.Metric{} if a.Spec.ResourcePolicy == nil || a.Spec.ResourcePolicy.ContainerPolicies == nil { return &metric.Family{ @@ -130,12 +166,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound", - Type: metric.Gauge, - Help: "Minimum resources the container can use before the VerticalPodAutoscaler updater evicts it.", - GenerateFunc: wrapVPAFunc(func(a *autoscaling.VerticalPodAutoscaler) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound", + "Minimum resources the container can use before the VerticalPodAutoscaler updater evicts it.", + metric.Gauge, + basemetrics.ALPHA, + "v2.9.0", + wrapVPAFunc(func(a *autoscaling.VerticalPodAutoscaler) *metric.Family { ms := []*metric.Metric{} if a.Status.Recommendation == nil || a.Status.Recommendation.ContainerRecommendations == nil { return &metric.Family{ @@ -150,12 +188,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound", - Type: metric.Gauge, - Help: "Maximum resources the container can use before the VerticalPodAutoscaler updater evicts it.", - GenerateFunc: wrapVPAFunc(func(a *autoscaling.VerticalPodAutoscaler) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound", + "Maximum resources the container can use before the VerticalPodAutoscaler updater evicts it.", + metric.Gauge, + basemetrics.ALPHA, + "v2.9.0", + wrapVPAFunc(func(a *autoscaling.VerticalPodAutoscaler) *metric.Family { ms := []*metric.Metric{} if a.Status.Recommendation == nil || a.Status.Recommendation.ContainerRecommendations == nil { return &metric.Family{ @@ -170,12 +210,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target", - Type: metric.Gauge, - Help: "Target resources the VerticalPodAutoscaler recommends for the container.", - GenerateFunc: wrapVPAFunc(func(a *autoscaling.VerticalPodAutoscaler) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target", + "Target resources the VerticalPodAutoscaler recommends for the container.", + metric.Gauge, + basemetrics.ALPHA, + "v2.9.0", + wrapVPAFunc(func(a *autoscaling.VerticalPodAutoscaler) *metric.Family { ms := []*metric.Metric{} if a.Status.Recommendation == nil || a.Status.Recommendation.ContainerRecommendations == nil { return &metric.Family{ @@ -189,12 +231,14 @@ var ( Metrics: ms, } }), - }, - { - Name: "kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget", - Type: metric.Gauge, - Help: "Target resources the VerticalPodAutoscaler recommends for the container ignoring bounds.", - GenerateFunc: wrapVPAFunc(func(a *autoscaling.VerticalPodAutoscaler) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget", + "Target resources the VerticalPodAutoscaler recommends for the container ignoring bounds.", + metric.Gauge, + basemetrics.ALPHA, + "v2.9.0", + wrapVPAFunc(func(a *autoscaling.VerticalPodAutoscaler) *metric.Family { ms := []*metric.Metric{} if a.Status.Recommendation == nil || a.Status.Recommendation.ContainerRecommendations == nil { return &metric.Family{ @@ -208,9 +252,9 @@ var ( Metrics: ms, } }), - }, + ), } -) +} func vpaResourcesToMetrics(containerName string, resources v1.ResourceList) []*metric.Metric { ms := []*metric.Metric{} @@ -245,23 +289,32 @@ func wrapVPAFunc(f func(*autoscaling.VerticalPodAutoscaler) *metric.Family) func metricFamily := f(vpa) targetRef := vpa.Spec.TargetRef + // targetRef was not a mandatory field, which can lead to a nil pointer exception here. + // However, we still want to expose metrics to be able: + // * to alert about VPA objects without target refs + // * to count the right amount of VPA objects in a cluster + if targetRef == nil { + targetRef = &autoscalingv1.CrossVersionObjectReference{} + } + for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descVerticalPodAutoscalerLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{vpa.Namespace, vpa.Name, targetRef.APIVersion, targetRef.Kind, targetRef.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descVerticalPodAutoscalerLabelsDefaultLabels, []string{vpa.Namespace, vpa.Name, targetRef.APIVersion, targetRef.Kind, targetRef.Name}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createVPAListWatchFunc(vpaClient vpaclientset.Interface) func(kubeClient clientset.Interface, ns string) cache.ListerWatcher { - return func(kubeClient clientset.Interface, ns string) cache.ListerWatcher { +func createVPAListWatchFunc(vpaClient vpaclientset.Interface) func(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { + return func(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return vpaClient.AutoscalingV1beta2().VerticalPodAutoscalers(ns).List(opts) + opts.FieldSelector = fieldSelector + return vpaClient.AutoscalingV1beta2().VerticalPodAutoscalers(ns).List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return vpaClient.AutoscalingV1beta2().VerticalPodAutoscalers(ns).Watch(opts) + opts.FieldSelector = fieldSelector + return vpaClient.AutoscalingV1beta2().VerticalPodAutoscalers(ns).Watch(context.TODO(), opts) }, } } diff --git a/internal/store/verticalpodautoscaler_test.go b/internal/store/verticalpodautoscaler_test.go index ddb8698c99..4c107b3c6c 100644 --- a/internal/store/verticalpodautoscaler_test.go +++ b/internal/store/verticalpodautoscaler_test.go @@ -19,25 +19,25 @@ package store import ( "testing" - k8sautoscaling "k8s.io/api/autoscaling/v1" + autoscalingv1 "k8s.io/api/autoscaling/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" autoscaling "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1beta2" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) func TestVPAStore(t *testing.T) { const metadata = ` - # HELP kube_verticalpodautoscaler_labels Kubernetes labels converted to Prometheus labels. - # HELP kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed Maximum resources the VerticalPodAutoscaler can set for containers matching the name. - # HELP kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed Minimum resources the VerticalPodAutoscaler can set for containers matching the name. - # HELP kube_verticalpodautoscaler_spec_updatepolicy_updatemode Update mode of the VerticalPodAutoscaler. - # HELP kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound Minimum resources the container can use before the VerticalPodAutoscaler updater evicts it. - # HELP kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target Target resources the VerticalPodAutoscaler recommends for the container. - # HELP kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget Target resources the VerticalPodAutoscaler recommends for the container ignoring bounds. - # HELP kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound Maximum resources the container can use before the VerticalPodAutoscaler updater evicts it. + # HELP kube_verticalpodautoscaler_labels (Deprecated since v2.9.0) Kubernetes labels converted to Prometheus labels. + # HELP kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed (Deprecated since v2.9.0) Maximum resources the VerticalPodAutoscaler can set for containers matching the name. + # HELP kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed (Deprecated since v2.9.0) Minimum resources the VerticalPodAutoscaler can set for containers matching the name. + # HELP kube_verticalpodautoscaler_spec_updatepolicy_updatemode (Deprecated since v2.9.0) Update mode of the VerticalPodAutoscaler. + # HELP kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound (Deprecated since v2.9.0) Minimum resources the container can use before the VerticalPodAutoscaler updater evicts it. + # HELP kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target (Deprecated since v2.9.0) Target resources the VerticalPodAutoscaler recommends for the container. + # HELP kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget (Deprecated since v2.9.0) Target resources the VerticalPodAutoscaler recommends for the container ignoring bounds. + # HELP kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound (Deprecated since v2.9.0) Maximum resources the container can use before the VerticalPodAutoscaler updater evicts it. # TYPE kube_verticalpodautoscaler_labels gauge # TYPE kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed gauge # TYPE kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed gauge @@ -69,8 +69,8 @@ func TestVPAStore(t *testing.T) { }, }, Spec: autoscaling.VerticalPodAutoscalerSpec{ - TargetRef: &k8sautoscaling.CrossVersionObjectReference{ - APIVersion: "extensions/v1beta1", + TargetRef: &autoscalingv1.CrossVersionObjectReference{ + APIVersion: "apps/v1", Kind: "Deployment", Name: "deployment1", }, @@ -102,23 +102,91 @@ func TestVPAStore(t *testing.T) { }, }, Want: metadata + ` - kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed{container="*",namespace="ns1",resource="cpu",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",unit="core",verticalpodautoscaler="vpa1"} 4 - kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed{container="*",namespace="ns1",resource="memory",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",unit="byte",verticalpodautoscaler="vpa1"} 8.589934592e+09 - kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed{container="*",namespace="ns1",resource="cpu",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",unit="core",verticalpodautoscaler="vpa1"} 1 - kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed{container="*",namespace="ns1",resource="memory",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",unit="byte",verticalpodautoscaler="vpa1"} 4.294967296e+09 - kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound{container="container1",namespace="ns1",resource="cpu",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",unit="core",verticalpodautoscaler="vpa1"} 1 - kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound{container="container1",namespace="ns1",resource="memory",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",unit="byte",verticalpodautoscaler="vpa1"} 4.294967296e+09 - kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target{container="container1",namespace="ns1",resource="cpu",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",unit="core",verticalpodautoscaler="vpa1"} 3 - kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target{container="container1",namespace="ns1",resource="memory",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",unit="byte",verticalpodautoscaler="vpa1"} 7.516192768e+09 - kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget{container="container1",namespace="ns1",resource="cpu",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",unit="core",verticalpodautoscaler="vpa1"} 6 - kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget{container="container1",namespace="ns1",resource="memory",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",unit="byte",verticalpodautoscaler="vpa1"} 1.073741824e+10 - kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound{container="container1",namespace="ns1",resource="cpu",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",unit="core",verticalpodautoscaler="vpa1"} 4 - kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound{container="container1",namespace="ns1",resource="memory",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",unit="byte",verticalpodautoscaler="vpa1"} 8.589934592e+09 - kube_verticalpodautoscaler_labels{label_app="foobar",namespace="ns1",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",verticalpodautoscaler="vpa1"} 1 - kube_verticalpodautoscaler_spec_updatepolicy_updatemode{namespace="ns1",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",update_mode="Auto",verticalpodautoscaler="vpa1"} 0 - kube_verticalpodautoscaler_spec_updatepolicy_updatemode{namespace="ns1",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",update_mode="Initial",verticalpodautoscaler="vpa1"} 0 - kube_verticalpodautoscaler_spec_updatepolicy_updatemode{namespace="ns1",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",update_mode="Off",verticalpodautoscaler="vpa1"} 0 - kube_verticalpodautoscaler_spec_updatepolicy_updatemode{namespace="ns1",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",update_mode="Recreate",verticalpodautoscaler="vpa1"} 1 + kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed{container="*",namespace="ns1",resource="cpu",target_api_version="apps/v1",target_kind="Deployment",target_name="deployment1",unit="core",verticalpodautoscaler="vpa1"} 4 + kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed{container="*",namespace="ns1",resource="memory",target_api_version="apps/v1",target_kind="Deployment",target_name="deployment1",unit="byte",verticalpodautoscaler="vpa1"} 8.589934592e+09 + kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed{container="*",namespace="ns1",resource="cpu",target_api_version="apps/v1",target_kind="Deployment",target_name="deployment1",unit="core",verticalpodautoscaler="vpa1"} 1 + kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed{container="*",namespace="ns1",resource="memory",target_api_version="apps/v1",target_kind="Deployment",target_name="deployment1",unit="byte",verticalpodautoscaler="vpa1"} 4.294967296e+09 + kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound{container="container1",namespace="ns1",resource="cpu",target_api_version="apps/v1",target_kind="Deployment",target_name="deployment1",unit="core",verticalpodautoscaler="vpa1"} 1 + kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound{container="container1",namespace="ns1",resource="memory",target_api_version="apps/v1",target_kind="Deployment",target_name="deployment1",unit="byte",verticalpodautoscaler="vpa1"} 4.294967296e+09 + kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target{container="container1",namespace="ns1",resource="cpu",target_api_version="apps/v1",target_kind="Deployment",target_name="deployment1",unit="core",verticalpodautoscaler="vpa1"} 3 + kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target{container="container1",namespace="ns1",resource="memory",target_api_version="apps/v1",target_kind="Deployment",target_name="deployment1",unit="byte",verticalpodautoscaler="vpa1"} 7.516192768e+09 + kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget{container="container1",namespace="ns1",resource="cpu",target_api_version="apps/v1",target_kind="Deployment",target_name="deployment1",unit="core",verticalpodautoscaler="vpa1"} 6 + kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget{container="container1",namespace="ns1",resource="memory",target_api_version="apps/v1",target_kind="Deployment",target_name="deployment1",unit="byte",verticalpodautoscaler="vpa1"} 1.073741824e+10 + kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound{container="container1",namespace="ns1",resource="cpu",target_api_version="apps/v1",target_kind="Deployment",target_name="deployment1",unit="core",verticalpodautoscaler="vpa1"} 4 + kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound{container="container1",namespace="ns1",resource="memory",target_api_version="apps/v1",target_kind="Deployment",target_name="deployment1",unit="byte",verticalpodautoscaler="vpa1"} 8.589934592e+09 + kube_verticalpodautoscaler_labels{namespace="ns1",target_api_version="apps/v1",target_kind="Deployment",target_name="deployment1",verticalpodautoscaler="vpa1"} 1 + kube_verticalpodautoscaler_spec_updatepolicy_updatemode{namespace="ns1",target_api_version="apps/v1",target_kind="Deployment",target_name="deployment1",update_mode="Auto",verticalpodautoscaler="vpa1"} 0 + kube_verticalpodautoscaler_spec_updatepolicy_updatemode{namespace="ns1",target_api_version="apps/v1",target_kind="Deployment",target_name="deployment1",update_mode="Initial",verticalpodautoscaler="vpa1"} 0 + kube_verticalpodautoscaler_spec_updatepolicy_updatemode{namespace="ns1",target_api_version="apps/v1",target_kind="Deployment",target_name="deployment1",update_mode="Off",verticalpodautoscaler="vpa1"} 0 + kube_verticalpodautoscaler_spec_updatepolicy_updatemode{namespace="ns1",target_api_version="apps/v1",target_kind="Deployment",target_name="deployment1",update_mode="Recreate",verticalpodautoscaler="vpa1"} 1 + `, + MetricNames: []string{ + "kube_verticalpodautoscaler_labels", + "kube_verticalpodautoscaler_spec_updatepolicy_updatemode", + "kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed", + "kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed", + "kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound", + "kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound", + "kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target", + "kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget", + }, + }, + { + Obj: &autoscaling.VerticalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 2, + Name: "vpa-without-target-ref", + Namespace: "ns2", + Labels: map[string]string{ + "app": "foobar", + }, + }, + Spec: autoscaling.VerticalPodAutoscalerSpec{ + UpdatePolicy: &autoscaling.PodUpdatePolicy{ + UpdateMode: &updateMode, + }, + ResourcePolicy: &autoscaling.PodResourcePolicy{ + ContainerPolicies: []autoscaling.ContainerResourcePolicy{ + { + ContainerName: "*", + MinAllowed: v1Resource("1", "4Gi"), + MaxAllowed: v1Resource("4", "8Gi"), + }, + }, + }, + }, + Status: autoscaling.VerticalPodAutoscalerStatus{ + Recommendation: &autoscaling.RecommendedPodResources{ + ContainerRecommendations: []autoscaling.RecommendedContainerResources{ + { + ContainerName: "container1", + LowerBound: v1Resource("1", "4Gi"), + UpperBound: v1Resource("4", "8Gi"), + Target: v1Resource("3", "7Gi"), + UncappedTarget: v1Resource("6", "10Gi"), + }, + }, + }, + }, + }, + Want: metadata + ` + kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed{container="*",namespace="ns2",resource="cpu",target_api_version="",target_kind="",target_name="",unit="core",verticalpodautoscaler="vpa-without-target-ref"} 4 + kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed{container="*",namespace="ns2",resource="memory",target_api_version="",target_kind="",target_name="",unit="byte",verticalpodautoscaler="vpa-without-target-ref"} 8.589934592e+09 + kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed{container="*",namespace="ns2",resource="cpu",target_api_version="",target_kind="",target_name="",unit="core",verticalpodautoscaler="vpa-without-target-ref"} 1 + kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed{container="*",namespace="ns2",resource="memory",target_api_version="",target_kind="",target_name="",unit="byte",verticalpodautoscaler="vpa-without-target-ref"} 4.294967296e+09 + kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound{container="container1",namespace="ns2",resource="cpu",target_api_version="",target_kind="",target_name="",unit="core",verticalpodautoscaler="vpa-without-target-ref"} 1 + kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound{container="container1",namespace="ns2",resource="memory",target_api_version="",target_kind="",target_name="",unit="byte",verticalpodautoscaler="vpa-without-target-ref"} 4.294967296e+09 + kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target{container="container1",namespace="ns2",resource="cpu",target_api_version="",target_kind="",target_name="",unit="core",verticalpodautoscaler="vpa-without-target-ref"} 3 + kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target{container="container1",namespace="ns2",resource="memory",target_api_version="",target_kind="",target_name="",unit="byte",verticalpodautoscaler="vpa-without-target-ref"} 7.516192768e+09 + kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget{container="container1",namespace="ns2",resource="cpu",target_api_version="",target_kind="",target_name="",unit="core",verticalpodautoscaler="vpa-without-target-ref"} 6 + kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget{container="container1",namespace="ns2",resource="memory",target_api_version="",target_kind="",target_name="",unit="byte",verticalpodautoscaler="vpa-without-target-ref"} 1.073741824e+10 + kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound{container="container1",namespace="ns2",resource="cpu",target_api_version="",target_kind="",target_name="",unit="core",verticalpodautoscaler="vpa-without-target-ref"} 4 + kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound{container="container1",namespace="ns2",resource="memory",target_api_version="",target_kind="",target_name="",unit="byte",verticalpodautoscaler="vpa-without-target-ref"} 8.589934592e+09 + kube_verticalpodautoscaler_labels{namespace="ns2",target_api_version="",target_kind="",target_name="",verticalpodautoscaler="vpa-without-target-ref"} 1 + kube_verticalpodautoscaler_spec_updatepolicy_updatemode{namespace="ns2",target_api_version="",target_kind="",target_name="",update_mode="Auto",verticalpodautoscaler="vpa-without-target-ref"} 0 + kube_verticalpodautoscaler_spec_updatepolicy_updatemode{namespace="ns2",target_api_version="",target_kind="",target_name="",update_mode="Initial",verticalpodautoscaler="vpa-without-target-ref"} 0 + kube_verticalpodautoscaler_spec_updatepolicy_updatemode{namespace="ns2",target_api_version="",target_kind="",target_name="",update_mode="Off",verticalpodautoscaler="vpa-without-target-ref"} 0 + kube_verticalpodautoscaler_spec_updatepolicy_updatemode{namespace="ns2",target_api_version="",target_kind="",target_name="",update_mode="Recreate",verticalpodautoscaler="vpa-without-target-ref"} 1 `, MetricNames: []string{ "kube_verticalpodautoscaler_labels", @@ -133,8 +201,8 @@ func TestVPAStore(t *testing.T) { }, } for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(vpaMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(vpaMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(vpaMetricFamilies(nil, nil)) + c.Headers = generator.ExtractMetricFamilyHeaders(vpaMetricFamilies(nil, nil)) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/store/volumeattachment.go b/internal/store/volumeattachment.go index f57050b743..22cdd8e963 100644 --- a/internal/store/volumeattachment.go +++ b/internal/store/volumeattachment.go @@ -17,14 +17,18 @@ limitations under the License. package store import ( + "context" + storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + basemetrics "k8s.io/component-base/metrics" - "k8s.io/kube-state-metrics/pkg/metric" + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) var ( @@ -32,13 +36,15 @@ var ( descVolumeAttachmentLabelsHelp = "Kubernetes labels converted to Prometheus labels." descVolumeAttachmentLabelsDefaultLabels = []string{"volumeattachment"} - volumeAttachmentMetricFamilies = []metric.FamilyGenerator{ - { - Name: descVolumeAttachmentLabelsName, - Type: metric.Gauge, - Help: descVolumeAttachmentLabelsHelp, - GenerateFunc: wrapVolumeAttachmentFunc(func(va *storagev1.VolumeAttachment) *metric.Family { - labelKeys, labelValues := kubeLabelsToPrometheusLabels(va.Labels) + volumeAttachmentMetricFamilies = []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + descVolumeAttachmentLabelsName, + descVolumeAttachmentLabelsHelp, + metric.Gauge, + basemetrics.ALPHA, + "", + wrapVolumeAttachmentFunc(func(va *storagev1.VolumeAttachment) *metric.Family { + labelKeys, labelValues := kubeMapToPrometheusLabels("label", va.Labels) return &metric.Family{ Metrics: []*metric.Metric{ { @@ -49,28 +55,32 @@ var ( }, } }), - }, - { - Name: "kube_volumeattachment_info", - Type: metric.Gauge, - Help: "Information about volumeattachment.", - GenerateFunc: wrapVolumeAttachmentFunc(func(va *storagev1.VolumeAttachment) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_volumeattachment_info", + "Information about volumeattachment.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapVolumeAttachmentFunc(func(va *storagev1.VolumeAttachment) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { - LabelKeys: []string{"attacher", "nodeName"}, + LabelKeys: []string{"attacher", "node"}, LabelValues: []string{va.Spec.Attacher, va.Spec.NodeName}, Value: 1, }, }, } }), - }, - { - Name: "kube_volumeattachment_created", - Type: metric.Gauge, - Help: "Unix creation timestamp", - GenerateFunc: wrapVolumeAttachmentFunc(func(va *storagev1.VolumeAttachment) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_volumeattachment_created", + "Unix creation timestamp", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapVolumeAttachmentFunc(func(va *storagev1.VolumeAttachment) *metric.Family { if !va.CreationTimestamp.IsZero() { m := metric.Metric{ LabelKeys: nil, @@ -81,12 +91,14 @@ var ( } return &metric.Family{Metrics: []*metric.Metric{}} }), - }, - { - Name: "kube_volumeattachment_spec_source_persistentvolume", - Type: metric.Gauge, - Help: "PersistentVolume source reference.", - GenerateFunc: wrapVolumeAttachmentFunc(func(va *storagev1.VolumeAttachment) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_volumeattachment_spec_source_persistentvolume", + "PersistentVolume source reference.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapVolumeAttachmentFunc(func(va *storagev1.VolumeAttachment) *metric.Family { if va.Spec.Source.PersistentVolumeName != nil { return &metric.Family{ Metrics: []*metric.Metric{ @@ -100,12 +112,14 @@ var ( } return &metric.Family{} }), - }, - { - Name: "kube_volumeattachment_status_attached", - Type: metric.Gauge, - Help: "Information about volumeattachment.", - GenerateFunc: wrapVolumeAttachmentFunc(func(va *storagev1.VolumeAttachment) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_volumeattachment_status_attached", + "Information about volumeattachment.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapVolumeAttachmentFunc(func(va *storagev1.VolumeAttachment) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{ { @@ -116,12 +130,14 @@ var ( }, } }), - }, - { - Name: "kube_volumeattachment_status_attachment_metadata", - Type: metric.Gauge, - Help: "volumeattachment metadata.", - GenerateFunc: wrapVolumeAttachmentFunc(func(va *storagev1.VolumeAttachment) *metric.Family { + ), + *generator.NewFamilyGeneratorWithStability( + "kube_volumeattachment_status_attachment_metadata", + "volumeattachment metadata.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapVolumeAttachmentFunc(func(va *storagev1.VolumeAttachment) *metric.Family { labelKeys, labelValues := mapToPrometheusLabels(va.Status.AttachmentMetadata, "metadata") return &metric.Family{ Metrics: []*metric.Metric{ @@ -133,7 +149,7 @@ var ( }, } }), - }, + ), } ) @@ -144,21 +160,20 @@ func wrapVolumeAttachmentFunc(f func(*storagev1.VolumeAttachment) *metric.Family metricFamily := f(va) for _, m := range metricFamily.Metrics { - m.LabelKeys = append(descVolumeAttachmentLabelsDefaultLabels, m.LabelKeys...) - m.LabelValues = append([]string{va.Name}, m.LabelValues...) + m.LabelKeys, m.LabelValues = mergeKeyValues(descVolumeAttachmentLabelsDefaultLabels, []string{va.Name}, m.LabelKeys, m.LabelValues) } return metricFamily } } -func createVolumeAttachmentListWatch(kubeClient clientset.Interface, _ string) cache.ListerWatcher { +func createVolumeAttachmentListWatch(kubeClient clientset.Interface, _ string, _ string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.StorageV1().VolumeAttachments().List(opts) + return kubeClient.StorageV1().VolumeAttachments().List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.StorageV1().VolumeAttachments().Watch(opts) + return kubeClient.StorageV1().VolumeAttachments().Watch(context.TODO(), opts) }, } } diff --git a/internal/store/volumeattachment_test.go b/internal/store/volumeattachment_test.go index d943a9947f..822cd21f31 100644 --- a/internal/store/volumeattachment_test.go +++ b/internal/store/volumeattachment_test.go @@ -22,7 +22,7 @@ import ( storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kube-state-metrics/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) func TestVolumeAttachmentStore(t *testing.T) { @@ -69,11 +69,11 @@ func TestVolumeAttachmentStore(t *testing.T) { }, }, Want: metadata + ` - kube_volumeattachment_info{attacher="cinder.csi.openstack.org",nodeName="node1",volumeattachment="csi-5ff16a1ad085261021e21c6cb3a6defb979a8794f25a4f90f6285664cff37224"} 1 + kube_volumeattachment_info{attacher="cinder.csi.openstack.org",node="node1",volumeattachment="csi-5ff16a1ad085261021e21c6cb3a6defb979a8794f25a4f90f6285664cff37224"} 1 kube_volumeattachment_labels{label_app="foobar",volumeattachment="csi-5ff16a1ad085261021e21c6cb3a6defb979a8794f25a4f90f6285664cff37224"} 1 kube_volumeattachment_spec_source_persistentvolume{volumeattachment="csi-5ff16a1ad085261021e21c6cb3a6defb979a8794f25a4f90f6285664cff37224",volumename="pvc-44f6ff3f-ba9b-49c4-9b95-8b01c4bd4bab"} 1 kube_volumeattachment_status_attached{volumeattachment="csi-5ff16a1ad085261021e21c6cb3a6defb979a8794f25a4f90f6285664cff37224"} 1 - kube_volumeattachment_status_attachment_metadata{metadata_DevicePath="/dev/sdd",volumeattachment="csi-5ff16a1ad085261021e21c6cb3a6defb979a8794f25a4f90f6285664cff37224"} 1 + kube_volumeattachment_status_attachment_metadata{metadata_device_path="/dev/sdd",volumeattachment="csi-5ff16a1ad085261021e21c6cb3a6defb979a8794f25a4f90f6285664cff37224"} 1 `, MetricNames: []string{ "kube_volumeattachment_labels", @@ -87,8 +87,8 @@ func TestVolumeAttachmentStore(t *testing.T) { } ) for i, c := range cases { - c.Func = metric.ComposeMetricGenFuncs(volumeAttachmentMetricFamilies) - c.Headers = metric.ExtractMetricFamilyHeaders(volumeAttachmentMetricFamilies) + c.Func = generator.ComposeMetricGenFuncs(volumeAttachmentMetricFamilies) + c.Headers = generator.ExtractMetricFamilyHeaders(volumeAttachmentMetricFamilies) if err := c.run(); err != nil { t.Errorf("unexpected collecting result in %vth run:\n%s", i, err) } diff --git a/internal/wrapper.go b/internal/wrapper.go new file mode 100644 index 0000000000..0a530d503a --- /dev/null +++ b/internal/wrapper.go @@ -0,0 +1,101 @@ +/* +Copyright 2022 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "context" + "errors" + "os" + "path/filepath" + "time" + + "github.com/fsnotify/fsnotify" + "github.com/spf13/viper" + "gopkg.in/yaml.v3" + "k8s.io/klog/v2" + + "k8s.io/kube-state-metrics/v2/pkg/app" + "k8s.io/kube-state-metrics/v2/pkg/options" +) + +// RunKubeStateMetricsWrapper is a wrapper around KSM, delegated to the root command. +func RunKubeStateMetricsWrapper(opts *options.Options) { + + KSMRunOrDie := func(ctx context.Context) { + if err := app.RunKubeStateMetricsWrapper(ctx, opts); err != nil { + klog.ErrorS(err, "Failed to run kube-state-metrics") + klog.FlushAndExit(klog.ExitFlushTimeout, 1) + } + } + + ctx, cancel := context.WithCancel(context.Background()) + if file := options.GetConfigFile(*opts); file != "" { + cfgViper := viper.New() + cfgViper.SetConfigType("yaml") + cfgViper.SetConfigFile(file) + if err := cfgViper.ReadInConfig(); err != nil { + if errors.Is(err, viper.ConfigFileNotFoundError{}) { + klog.ErrorS(err, "Options configuration file not found", "file", file) + } else { + klog.ErrorS(err, "Error reading options configuration file", "file", file) + } + klog.FlushAndExit(klog.ExitFlushTimeout, 1) + } + cfgViper.OnConfigChange(func(e fsnotify.Event) { + klog.InfoS("Changes detected", "name", e.Name) + cancel() + // Wait for the ports to be released. + <-time.After(3 * time.Second) + ctx, cancel = context.WithCancel(context.Background()) + go KSMRunOrDie(ctx) + }) + cfgViper.WatchConfig() + + // Merge configFile values with opts so we get the CustomResourceConfigFile from config as well + configFile, err := os.ReadFile(filepath.Clean(file)) + if err != nil { + klog.ErrorS(err, "failed to read options configuration file", "file", file) + } + + yaml.Unmarshal(configFile, opts) + } + if opts.CustomResourceConfigFile != "" { + crcViper := viper.New() + crcViper.SetConfigType("yaml") + crcViper.SetConfigFile(opts.CustomResourceConfigFile) + if err := crcViper.ReadInConfig(); err != nil { + if errors.Is(err, viper.ConfigFileNotFoundError{}) { + klog.ErrorS(err, "Custom resource configuration file not found", "file", opts.CustomResourceConfigFile) + } else { + klog.ErrorS(err, "Error reading Custom resource configuration file", "file", opts.CustomResourceConfigFile) + } + klog.FlushAndExit(klog.ExitFlushTimeout, 1) + } + crcViper.OnConfigChange(func(e fsnotify.Event) { + klog.InfoS("Changes detected", "name", e.Name) + cancel() + // Wait for the ports to be released. + <-time.After(3 * time.Second) + ctx, cancel = context.WithCancel(context.Background()) + go KSMRunOrDie(ctx) + }) + crcViper.WatchConfig() + } + klog.InfoS("Starting kube-state-metrics") + KSMRunOrDie(ctx) + select {} +} diff --git a/jsonnet/kube-state-metrics-mixin/alerts.libsonnet b/jsonnet/kube-state-metrics-mixin/alerts.libsonnet index 0d30a99823..396f910912 100644 --- a/jsonnet/kube-state-metrics-mixin/alerts.libsonnet +++ b/jsonnet/kube-state-metrics-mixin/alerts.libsonnet @@ -1,7 +1,4 @@ { - _config+:: { - kubeStateMetricsSelector: error 'must provide selector for kube-state-metrics', - }, prometheusAlerts+:: { groups+: [ { @@ -20,7 +17,8 @@ severity: 'critical', }, annotations: { - message: 'kube-state-metrics is experiencing errors at an elevated rate in list operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all.', + summary: 'kube-state-metrics is experiencing errors in list operations.', + description: 'kube-state-metrics is experiencing errors at an elevated rate in list operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all.', }, }, { @@ -36,7 +34,45 @@ severity: 'critical', }, annotations: { - message: 'kube-state-metrics is experiencing errors at an elevated rate in watch operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all.', + summary: 'kube-state-metrics is experiencing errors in watch operations.', + description: 'kube-state-metrics is experiencing errors at an elevated rate in watch operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all.', + }, + }, + { + alert: 'KubeStateMetricsShardingMismatch', + // + expr: ||| + stdvar (kube_state_metrics_total_shards{%(kubeStateMetricsSelector)s}) != 0 + ||| % $._config, + 'for': '15m', + labels: { + severity: 'critical', + }, + annotations: { + summary: 'kube-state-metrics sharding is misconfigured.', + description: 'kube-state-metrics pods are running with different --total-shards configuration, some Kubernetes objects may be exposed multiple times or not exposed at all.', + }, + }, + { + alert: 'KubeStateMetricsShardsMissing', + // Each shard ordinal is assigned a binary position (2^ordinal) and we compute a sum of those. + // This sum is compared to the expected number (2^total_shards - 1). + // Result of zero all shards are being scraped, anything else indicates an issue. + // A handy side effect of this computation is the result indicates what ordinals are missing. + // Eg. a result of "5" decimal, which translates to binary "101", means shards #0 and #2 are not available. + expr: ||| + 2^max(kube_state_metrics_total_shards{%(kubeStateMetricsSelector)s}) - 1 + - + sum( 2 ^ max by (shard_ordinal) (kube_state_metrics_shard_ordinal{%(kubeStateMetricsSelector)s}) ) + != 0 + ||| % $._config, + 'for': '15m', + labels: { + severity: 'critical', + }, + annotations: { + summary: 'kube-state-metrics shards are missing.', + description: 'kube-state-metrics shards are missing, some Kubernetes objects are not being exposed.', }, }, ], diff --git a/jsonnet/kube-state-metrics-mixin/config.libsonnet b/jsonnet/kube-state-metrics-mixin/config.libsonnet new file mode 100644 index 0000000000..8a5402c693 --- /dev/null +++ b/jsonnet/kube-state-metrics-mixin/config.libsonnet @@ -0,0 +1,6 @@ +{ + _config+:: { + // Select the metrics coming from the kube state metrics. + kubeStateMetricsSelector: 'job="kube-state-metrics"', + }, +} diff --git a/jsonnet/kube-state-metrics-mixin/mixin.libsonnet b/jsonnet/kube-state-metrics-mixin/mixin.libsonnet index d06ee2fd19..95efe331f7 100644 --- a/jsonnet/kube-state-metrics-mixin/mixin.libsonnet +++ b/jsonnet/kube-state-metrics-mixin/mixin.libsonnet @@ -1 +1,2 @@ +(import 'config.libsonnet') + (import 'alerts.libsonnet') diff --git a/jsonnet/kube-state-metrics/jsonnetfile.json b/jsonnet/kube-state-metrics/jsonnetfile.json index d31ac11001..677d98118f 100644 --- a/jsonnet/kube-state-metrics/jsonnetfile.json +++ b/jsonnet/kube-state-metrics/jsonnetfile.json @@ -1,14 +1,3 @@ { - "dependencies": [ - { - "name": "ksonnet", - "source": { - "git": { - "remote": "https://github.com/ksonnet/ksonnet-lib", - "subdir": "" - } - }, - "version": "master" - } - ] + "dependencies": [] } diff --git a/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet b/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet index 0f545fc7dc..d562d02c3b 100644 --- a/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet +++ b/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet @@ -1,5 +1,3 @@ -local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; - { local ksm = self, name:: error 'must set namespace', @@ -9,7 +7,11 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; commonLabels:: { 'app.kubernetes.io/name': 'kube-state-metrics', - 'app.kubernetes.io/version': 'v' + ksm.version, + 'app.kubernetes.io/version': ksm.version, + }, + + extraRecommendedLabels:: { + 'app.kubernetes.io/component': 'exporter', }, podLabels:: { @@ -19,237 +21,319 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; }, clusterRoleBinding: - local clusterRoleBinding = k.rbac.v1.clusterRoleBinding; - - clusterRoleBinding.new() + - clusterRoleBinding.mixin.metadata.withName(ksm.name) + - clusterRoleBinding.mixin.metadata.withLabels(ksm.commonLabels) + - clusterRoleBinding.mixin.roleRef.withApiGroup('rbac.authorization.k8s.io') + - clusterRoleBinding.mixin.roleRef.withName(ksm.name) + - clusterRoleBinding.mixin.roleRef.mixinInstance({ kind: 'ClusterRole' }) + - clusterRoleBinding.withSubjects([{ kind: 'ServiceAccount', name: ksm.name, namespace: ksm.namespace }]), + { + apiVersion: 'rbac.authorization.k8s.io/v1', + kind: 'ClusterRoleBinding', + metadata: { + name: ksm.name, + labels: ksm.commonLabels + ksm.extraRecommendedLabels, + }, + roleRef: { + apiGroup: 'rbac.authorization.k8s.io', + kind: 'ClusterRole', + name: ksm.name, + }, + subjects: [{ + kind: 'ServiceAccount', + name: ksm.name, + namespace: ksm.namespace, + }], + }, clusterRole: - local clusterRole = k.rbac.v1.clusterRole; - local rulesType = clusterRole.rulesType; - local rules = [ - rulesType.new() + - rulesType.withApiGroups(['']) + - rulesType.withResources([ - 'configmaps', - 'secrets', - 'nodes', - 'pods', - 'services', - 'resourcequotas', - 'replicationcontrollers', - 'limitranges', - 'persistentvolumeclaims', - 'persistentvolumes', - 'namespaces', - 'endpoints', - ]) + - rulesType.withVerbs(['list', 'watch']), - - rulesType.new() + - rulesType.withApiGroups(['extensions']) + - rulesType.withResources([ - 'daemonsets', - 'deployments', - 'replicasets', - 'ingresses', - ]) + - rulesType.withVerbs(['list', 'watch']), - - rulesType.new() + - rulesType.withApiGroups(['apps']) + - rulesType.withResources([ - 'statefulsets', - 'daemonsets', - 'deployments', - 'replicasets', - ]) + - rulesType.withVerbs(['list', 'watch']), - - rulesType.new() + - rulesType.withApiGroups(['batch']) + - rulesType.withResources([ - 'cronjobs', - 'jobs', - ]) + - rulesType.withVerbs(['list', 'watch']), - - rulesType.new() + - rulesType.withApiGroups(['autoscaling']) + - rulesType.withResources([ - 'horizontalpodautoscalers', - ]) + - rulesType.withVerbs(['list', 'watch']), - - rulesType.new() + - rulesType.withApiGroups(['authentication.k8s.io']) + - rulesType.withResources([ - 'tokenreviews', - ]) + - rulesType.withVerbs(['create']), - - rulesType.new() + - rulesType.withApiGroups(['authorization.k8s.io']) + - rulesType.withResources([ - 'subjectaccessreviews', - ]) + - rulesType.withVerbs(['create']), - - rulesType.new() + - rulesType.withApiGroups(['policy']) + - rulesType.withResources([ - 'poddisruptionbudgets', - ]) + - rulesType.withVerbs(['list', 'watch']), - - rulesType.new() + - rulesType.withApiGroups(['certificates.k8s.io']) + - rulesType.withResources([ - 'certificatesigningrequests', - ]) + - rulesType.withVerbs(['list', 'watch']), - - rulesType.new() + - rulesType.withApiGroups(['storage.k8s.io']) + - rulesType.withResources([ - 'storageclasses', - 'volumeattachments', - ]) + - rulesType.withVerbs(['list', 'watch']), - - rulesType.new() + - rulesType.withApiGroups(['admissionregistration.k8s.io']) + - rulesType.withResources([ - 'mutatingwebhookconfigurations', - 'validatingwebhookconfigurations', - ]) + - rulesType.withVerbs(['list', 'watch']), - - rulesType.new() + - rulesType.withApiGroups(['networking.k8s.io']) + - rulesType.withResources([ - 'networkpolicies', - ]) + - rulesType.withVerbs(['list', 'watch']), - ]; - - clusterRole.new() + - clusterRole.mixin.metadata.withName(ksm.name) + - clusterRole.mixin.metadata.withLabels(ksm.commonLabels) + - clusterRole.withRules(rules), + { + apiGroups: [''], + resources: [ + 'configmaps', + 'secrets', + 'nodes', + 'pods', + 'services', + 'serviceaccounts', + 'resourcequotas', + 'replicationcontrollers', + 'limitranges', + 'persistentvolumeclaims', + 'persistentvolumes', + 'namespaces', + 'endpoints', + ], + verbs: ['list', 'watch'], + }, + { + apiGroups: ['apps'], + resources: [ + 'statefulsets', + 'daemonsets', + 'deployments', + 'replicasets', + ], + verbs: ['list', 'watch'], + }, + { + apiGroups: ['batch'], + resources: [ + 'cronjobs', + 'jobs', + ], + verbs: ['list', 'watch'], + }, + { + apiGroups: ['autoscaling'], + resources: [ + 'horizontalpodautoscalers', + ], + verbs: ['list', 'watch'], + }, + { + apiGroups: ['authentication.k8s.io'], + resources: [ + 'tokenreviews', + ], + verbs: ['create'], + }, + { + apiGroups: ['authorization.k8s.io'], + resources: [ + 'subjectaccessreviews', + ], + verbs: ['create'], + }, + { + apiGroups: ['policy'], + resources: [ + 'poddisruptionbudgets', + ], + verbs: ['list', 'watch'], + }, + { + apiGroups: ['certificates.k8s.io'], + resources: [ + 'certificatesigningrequests', + ], + verbs: ['list', 'watch'], + }, + { + apiGroups: ['discovery.k8s.io'], + resources: [ + 'endpointslices', + ], + verbs: ['list', 'watch'], + }, + { + apiGroups: ['storage.k8s.io'], + resources: [ + 'storageclasses', + 'volumeattachments', + ], + verbs: ['list', 'watch'], + }, + { + apiGroups: ['admissionregistration.k8s.io'], + resources: [ + 'mutatingwebhookconfigurations', + 'validatingwebhookconfigurations', + ], + verbs: ['list', 'watch'], + }, + { + apiGroups: ['networking.k8s.io'], + resources: [ + 'networkpolicies', + 'ingressclasses', + 'ingresses', + ], + verbs: ['list', 'watch'], + }, + { + apiGroups: ['coordination.k8s.io'], + resources: [ + 'leases', + ], + verbs: ['list', 'watch'], + }, + { + apiGroups: ['rbac.authorization.k8s.io'], + resources: [ + 'clusterrolebindings', + 'clusterroles', + 'rolebindings', + 'roles', + ], + verbs: ['list', 'watch'], + }, + ]; + + { + apiVersion: 'rbac.authorization.k8s.io/v1', + kind: 'ClusterRole', + metadata: { + name: ksm.name, + labels: ksm.commonLabels + ksm.extraRecommendedLabels, + }, + rules: rules, + }, deployment: - local deployment = k.apps.v1.deployment; - local container = deployment.mixin.spec.template.spec.containersType; - local volume = deployment.mixin.spec.template.spec.volumesType; - local containerPort = container.portsType; - local containerVolumeMount = container.volumeMountsType; - local podSelector = deployment.mixin.spec.template.spec.selectorType; - - local c = - container.new('kube-state-metrics', ksm.image) + - container.withPorts([ - containerPort.newNamed(8080, 'http-metrics'), - containerPort.newNamed(8081, 'telemetry'), - ]) + - container.mixin.livenessProbe.httpGet.withPath('/healthz') + - container.mixin.livenessProbe.httpGet.withPort(8080) + - container.mixin.livenessProbe.withInitialDelaySeconds(5) + - container.mixin.livenessProbe.withTimeoutSeconds(5) + - container.mixin.readinessProbe.httpGet.withPath('/') + - container.mixin.readinessProbe.httpGet.withPort(8081) + - container.mixin.readinessProbe.withInitialDelaySeconds(5) + - container.mixin.readinessProbe.withTimeoutSeconds(5); - - deployment.new(ksm.name, 1, c, ksm.commonLabels) + - deployment.mixin.metadata.withNamespace(ksm.namespace) + - deployment.mixin.metadata.withLabels(ksm.commonLabels) + - deployment.mixin.spec.selector.withMatchLabels(ksm.podLabels) + - deployment.mixin.spec.template.spec.withNodeSelector({ 'kubernetes.io/os': 'linux' }) + - deployment.mixin.spec.template.spec.withServiceAccountName(ksm.name), + local c = { + name: 'kube-state-metrics', + image: ksm.image, + ports: [ + { name: 'http-metrics', containerPort: 8080 }, + { name: 'telemetry', containerPort: 8081 }, + ], + securityContext: { + runAsUser: 65534, + allowPrivilegeEscalation: false, + readOnlyRootFilesystem: true, + capabilities: { drop: ['ALL'] }, + }, + livenessProbe: { timeoutSeconds: 5, initialDelaySeconds: 5, httpGet: { + port: 8080, + path: '/healthz', + } }, + readinessProbe: { timeoutSeconds: 5, initialDelaySeconds: 5, httpGet: { + port: 8081, + path: '/', + } }, + }; + + { + apiVersion: 'apps/v1', + kind: 'Deployment', + metadata: { + name: ksm.name, + namespace: ksm.namespace, + labels: ksm.commonLabels + ksm.extraRecommendedLabels, + }, + spec: { + replicas: 1, + selector: { matchLabels: ksm.podLabels }, + template: { + metadata: { + labels: ksm.commonLabels + ksm.extraRecommendedLabels, + }, + spec: { + containers: [c], + serviceAccountName: ksm.serviceAccount.metadata.name, + automountServiceAccountToken: true, + nodeSelector: { 'kubernetes.io/os': 'linux' }, + }, + }, + }, + }, serviceAccount: - local serviceAccount = k.core.v1.serviceAccount; - - serviceAccount.new(ksm.name) + - serviceAccount.mixin.metadata.withNamespace(ksm.namespace) + - serviceAccount.mixin.metadata.withLabels(ksm.commonLabels), + { + apiVersion: 'v1', + kind: 'ServiceAccount', + metadata: { + name: ksm.name, + namespace: ksm.namespace, + labels: ksm.commonLabels + ksm.extraRecommendedLabels, + }, + automountServiceAccountToken: false, + }, service: - local service = k.core.v1.service; - local servicePort = service.mixin.spec.portsType; - - local ksmServicePortMain = servicePort.newNamed('http-metrics', 8080, 'http-metrics'); - local ksmServicePortSelf = servicePort.newNamed('telemetry', 8081, 'telemetry'); - - service.new(ksm.name, ksm.podLabels, [ksmServicePortMain, ksmServicePortSelf]) + - service.mixin.metadata.withNamespace(ksm.namespace) + - service.mixin.metadata.withLabels(ksm.commonLabels) + - service.mixin.spec.withClusterIp('None'), + { + apiVersion: 'v1', + kind: 'Service', + metadata: { + name: ksm.name, + namespace: ksm.namespace, + labels: ksm.commonLabels + ksm.extraRecommendedLabels, + }, + spec: { + clusterIP: 'None', + selector: ksm.podLabels, + ports: [ + { name: 'http-metrics', port: 8080, targetPort: 'http-metrics' }, + { name: 'telemetry', port: 8081, targetPort: 'telemetry' }, + ], + }, + }, autosharding:: { role: - local role = k.rbac.v1.role; - local rulesType = role.rulesType; - - local rules = [ - rulesType.new() + - rulesType.withApiGroups(['']) + - rulesType.withResources(['pods']) + - rulesType.withVerbs(['get']), - rulesType.new() + - rulesType.withApiGroups(['apps']) + - rulesType.withResources(['statefulsets']) + - rulesType.withResourceNames([ksm.name]) + - rulesType.withVerbs(['get']), - ]; - - role.new() + - role.mixin.metadata.withName(ksm.name) + - role.mixin.metadata.withNamespace(ksm.namespace) + - role.mixin.metadata.withLabels(ksm.commonLabels) + - role.withRules(rules), + { + apiVersion: 'rbac.authorization.k8s.io/v1', + kind: 'Role', + metadata: { + name: ksm.name, + namespace: ksm.namespace, + labels: ksm.commonLabels + ksm.extraRecommendedLabels, + }, + rules: [{ + apiGroups: [''], + resources: ['pods'], + verbs: ['get'], + }, { + apiGroups: ['apps'], + resourceNames: ['kube-state-metrics'], + resources: ['statefulsets'], + verbs: ['get'], + }], + }, roleBinding: - local roleBinding = k.rbac.v1.roleBinding; - - roleBinding.new() + - roleBinding.mixin.metadata.withName(ksm.name) + - roleBinding.mixin.metadata.withLabels(ksm.commonLabels) + - roleBinding.mixin.roleRef.withApiGroup('rbac.authorization.k8s.io') + - roleBinding.mixin.roleRef.withName(ksm.name) + - roleBinding.mixin.roleRef.mixinInstance({ kind: 'Role' }) + - roleBinding.withSubjects([{ kind: 'ServiceAccount', name: ksm.name }]), + { + apiVersion: 'rbac.authorization.k8s.io/v1', + kind: 'RoleBinding', + metadata: { + name: ksm.name, + namespace: ksm.namespace, + labels: ksm.commonLabels + ksm.extraRecommendedLabels, + }, + roleRef: { + apiGroup: 'rbac.authorization.k8s.io', + kind: 'Role', + name: 'kube-state-metrics', + }, + subjects: [{ + kind: 'ServiceAccount', + name: ksm.serviceAccount.metadata.name, + }], + }, statefulset: - local statefulset = k.apps.v1.statefulSet; - local container = statefulset.mixin.spec.template.spec.containersType; - local containerEnv = container.envType; - - local c = ksm.deployment.spec.template.spec.containers[0] + - container.withArgs([ - '--pod=$(POD_NAME)', - '--pod-namespace=$(POD_NAMESPACE)', - ]) + - container.withEnv([ - containerEnv.new('POD_NAME') + - containerEnv.mixin.valueFrom.fieldRef.withFieldPath('metadata.name'), - containerEnv.new('POD_NAMESPACE') + - containerEnv.mixin.valueFrom.fieldRef.withFieldPath('metadata.namespace'), - ]); - - statefulset.new(ksm.name, 2, c, [], ksm.commonLabels) + - statefulset.mixin.metadata.withNamespace(ksm.namespace) + - statefulset.mixin.metadata.withLabels(ksm.commonLabels) + - statefulset.mixin.spec.withServiceName(ksm.service.metadata.name) + - statefulset.mixin.spec.selector.withMatchLabels(ksm.podLabels) + - statefulset.mixin.spec.template.spec.withNodeSelector({ 'kubernetes.io/os': 'linux' }) + - statefulset.mixin.spec.template.spec.withServiceAccountName(ksm.name), + // extending the default container from above + local c = ksm.deployment.spec.template.spec.containers[0] { + args: [ + '--pod=$(POD_NAME)', + '--pod-namespace=$(POD_NAMESPACE)', + ], + env: [ + { name: 'POD_NAME', valueFrom: { fieldRef: { fieldPath: 'metadata.name' } } }, + { name: 'POD_NAMESPACE', valueFrom: { fieldRef: { fieldPath: 'metadata.namespace' } } }, + ], + }; + + { + apiVersion: 'apps/v1', + kind: 'StatefulSet', + metadata: { + name: ksm.name, + namespace: ksm.namespace, + labels: ksm.commonLabels + ksm.extraRecommendedLabels, + }, + spec: { + replicas: 2, + selector: { matchLabels: ksm.podLabels }, + serviceName: ksm.service.metadata.name, + template: { + metadata: { + labels: ksm.commonLabels + ksm.extraRecommendedLabels, + }, + spec: { + containers: [c], + serviceAccountName: ksm.serviceAccount.metadata.name, + automountServiceAccountToken: true, + nodeSelector: { 'kubernetes.io/os': 'linux' }, + }, + }, + }, + }, } + { service: ksm.service, serviceAccount: ksm.serviceAccount, diff --git a/main.go b/main.go index 9669001212..04fdad5a68 100644 --- a/main.go +++ b/main.go @@ -17,248 +17,27 @@ limitations under the License. package main import ( - "context" - "fmt" - "log" - "net" - "net/http" - "net/http/pprof" - "os" - "strconv" + "github.com/spf13/cobra" + "k8s.io/klog/v2" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" - vpaclientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned" - clientset "k8s.io/client-go/kubernetes" - _ "k8s.io/client-go/plugin/pkg/client/auth" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/klog" - - "k8s.io/kube-state-metrics/internal/store" - "k8s.io/kube-state-metrics/pkg/metricshandler" - "k8s.io/kube-state-metrics/pkg/options" - "k8s.io/kube-state-metrics/pkg/util/proc" - "k8s.io/kube-state-metrics/pkg/version" - "k8s.io/kube-state-metrics/pkg/whiteblacklist" -) - -const ( - metricsPath = "/metrics" - healthzPath = "/healthz" + "k8s.io/kube-state-metrics/v2/internal" + "k8s.io/kube-state-metrics/v2/pkg/options" ) -// promLogger implements promhttp.Logger -type promLogger struct{} - -func (pl promLogger) Println(v ...interface{}) { - klog.Error(v...) -} - func main() { opts := options.NewOptions() - opts.AddFlags() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - err := opts.Parse() - if err != nil { - klog.Fatalf("Error: %s", err) - } - - if opts.Version { - fmt.Printf("%#v\n", version.GetVersion()) - os.Exit(0) - } - - if opts.Help { - opts.Usage() - os.Exit(0) - } - storeBuilder := store.NewBuilder() - - ksmMetricsRegistry := prometheus.NewRegistry() - storeBuilder.WithMetrics(ksmMetricsRegistry) - - var collectors []string - if len(opts.Collectors) == 0 { - klog.Info("Using default collectors") - collectors = options.DefaultCollectors.AsSlice() - } else { - klog.Infof("Using collectors %s", opts.Collectors.String()) - collectors = opts.Collectors.AsSlice() - } - - if err := storeBuilder.WithEnabledResources(collectors); err != nil { - klog.Fatalf("Failed to set up collectors: %v", err) - } - - if len(opts.Namespaces) == 0 { - klog.Info("Using all namespace") - storeBuilder.WithNamespaces(options.DefaultNamespaces) - } else { - if opts.Namespaces.IsAllNamespaces() { - klog.Info("Using all namespace") - } else { - klog.Infof("Using %s namespaces", opts.Namespaces) - } - storeBuilder.WithNamespaces(opts.Namespaces) - } - - whiteBlackList, err := whiteblacklist.New(opts.MetricWhitelist, opts.MetricBlacklist) - if err != nil { - klog.Fatal(err) - } - - if opts.DisablePodNonGenericResourceMetrics { - whiteBlackList.Exclude([]string{ - "kube_pod_container_resource_requests_cpu_cores", - "kube_pod_container_resource_requests_memory_bytes", - "kube_pod_container_resource_limits_cpu_cores", - "kube_pod_container_resource_limits_memory_bytes", - }) - } - - if opts.DisableNodeNonGenericResourceMetrics { - whiteBlackList.Exclude([]string{ - "kube_node_status_capacity_cpu_cores", - "kube_node_status_capacity_memory_bytes", - "kube_node_status_capacity_pods", - "kube_node_status_allocatable_cpu_cores", - "kube_node_status_allocatable_memory_bytes", - "kube_node_status_allocatable_pods", - }) - } - - err = whiteBlackList.Parse() - if err != nil { - klog.Fatalf("error initializing the whiteblack list : %v", err) + cmd := options.InitCommand + cmd.Run = func(cmd *cobra.Command, args []string) { + internal.RunKubeStateMetricsWrapper(opts) } + opts.AddFlags(cmd) - klog.Infof("metric white-blacklisting: %v", whiteBlackList.Status()) - - storeBuilder.WithWhiteBlackList(whiteBlackList) - - proc.StartReaper() - - kubeClient, vpaClient, err := createKubeClient(opts.Apiserver, opts.Kubeconfig) - if err != nil { - klog.Fatalf("Failed to create client: %v", err) + if err := opts.Parse(); err != nil { + klog.FlushAndExit(klog.ExitFlushTimeout, 1) } - storeBuilder.WithKubeClient(kubeClient) - storeBuilder.WithVPAClient(vpaClient) - storeBuilder.WithSharding(opts.Shard, opts.TotalShards) - - ksmMetricsRegistry.MustRegister( - prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}), - prometheus.NewGoCollector(), - ) - go telemetryServer(ksmMetricsRegistry, opts.TelemetryHost, opts.TelemetryPort) - - serveMetrics(ctx, kubeClient, storeBuilder, opts, opts.Host, opts.Port, opts.EnableGZIPEncoding) -} -func createKubeClient(apiserver string, kubeconfig string) (clientset.Interface, vpaclientset.Interface, error) { - config, err := clientcmd.BuildConfigFromFlags(apiserver, kubeconfig) - if err != nil { - return nil, nil, err - } - - config.UserAgent = version.GetVersion().String() - config.AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json" - config.ContentType = "application/vnd.kubernetes.protobuf" - - kubeClient, err := clientset.NewForConfig(config) - if err != nil { - return nil, nil, err - } - - vpaClient, err := vpaclientset.NewForConfig(config) - if err != nil { - return nil, nil, err - } - // Informers don't seem to do a good job logging error messages when it - // can't reach the server, making debugging hard. This makes it easier to - // figure out if apiserver is configured incorrectly. - klog.Infof("Testing communication with server") - v, err := kubeClient.Discovery().ServerVersion() - if err != nil { - return nil, nil, errors.Wrap(err, "error while trying to communicate with apiserver") + if err := opts.Validate(); err != nil { + klog.ErrorS(err, "Validating options error") + klog.FlushAndExit(klog.ExitFlushTimeout, 1) } - klog.Infof("Running with Kubernetes cluster version: v%s.%s. git version: %s. git tree state: %s. commit: %s. platform: %s", - v.Major, v.Minor, v.GitVersion, v.GitTreeState, v.GitCommit, v.Platform) - klog.Infof("Communication with server successful") - - return kubeClient, vpaClient, nil -} - -func telemetryServer(registry prometheus.Gatherer, host string, port int) { - // Address to listen on for web interface and telemetry - listenAddress := net.JoinHostPort(host, strconv.Itoa(port)) - - klog.Infof("Starting kube-state-metrics self metrics server: %s", listenAddress) - - mux := http.NewServeMux() - - // Add metricsPath - mux.Handle(metricsPath, promhttp.HandlerFor(registry, promhttp.HandlerOpts{ErrorLog: promLogger{}})) - // Add index - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(` - Kube-State-Metrics Metrics Server - -

Kube-State-Metrics Metrics

- - - `)) - }) - log.Fatal(http.ListenAndServe(listenAddress, mux)) -} - -func serveMetrics(ctx context.Context, kubeClient clientset.Interface, storeBuilder *store.Builder, opts *options.Options, host string, port int, enableGZIPEncoding bool) { - // Address to listen on for web interface and telemetry - listenAddress := net.JoinHostPort(host, strconv.Itoa(port)) - - klog.Infof("Starting metrics server: %s", listenAddress) - - mux := http.NewServeMux() - - // TODO: This doesn't belong into serveMetrics - mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index)) - mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline)) - mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile)) - mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol)) - mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace)) - - m := metricshandler.New( - opts, - kubeClient, - storeBuilder, - enableGZIPEncoding, - ) - go m.Run(ctx) - mux.Handle(metricsPath, m) - - // Add healthzPath - mux.HandleFunc(healthzPath, func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Write([]byte(http.StatusText(http.StatusOK))) - }) - // Add index - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(` - Kube Metrics Server - -

Kube Metrics

- - - `)) - }) - log.Fatal(http.ListenAndServe(listenAddress, mux)) } diff --git a/main_test.go b/main_test.go deleted file mode 100644 index b0ee379389..0000000000 --- a/main_test.go +++ /dev/null @@ -1,628 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "bytes" - "context" - "io/ioutil" - "net/http/httptest" - "sort" - "strconv" - "strings" - "testing" - "time" - - "k8s.io/kube-state-metrics/internal/store" - "k8s.io/kube-state-metrics/pkg/metricshandler" - "k8s.io/kube-state-metrics/pkg/options" - "k8s.io/kube-state-metrics/pkg/whiteblacklist" - - "github.com/prometheus/client_golang/prometheus" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/fake" -) - -func BenchmarkKubeStateMetrics(b *testing.B) { - fixtureMultiplier := 1000 - requestCount := 1000 - - b.Logf( - "starting kube-state-metrics benchmark with fixtureMultiplier %v and requestCount %v", - fixtureMultiplier, - requestCount, - ) - - kubeClient := fake.NewSimpleClientset() - - if err := injectFixtures(kubeClient, fixtureMultiplier); err != nil { - b.Errorf("error injecting resources: %v", err) - } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - reg := prometheus.NewRegistry() - - builder := store.NewBuilder() - builder.WithMetrics(reg) - builder.WithEnabledResources(options.DefaultCollectors.AsSlice()) - builder.WithKubeClient(kubeClient) - builder.WithSharding(0, 1) - builder.WithContext(ctx) - builder.WithNamespaces(options.DefaultNamespaces) - - l, err := whiteblacklist.New(map[string]struct{}{}, map[string]struct{}{}) - if err != nil { - b.Fatal(err) - } - builder.WithWhiteBlackList(l) - - // This test is not suitable to be compared in terms of time, as it includes - // a one second wait. Use for memory allocation comparisons, profiling, ... - handler := metricshandler.New(&options.Options{}, kubeClient, builder, false) - b.Run("GenerateMetrics", func(b *testing.B) { - handler.ConfigureSharding(ctx, 0, 1) - - // Wait for caches to fill - time.Sleep(time.Second) - }) - - req := httptest.NewRequest("GET", "http://localhost:8080/metrics", nil) - - b.Run("MakeRequests", func(b *testing.B) { - var accumulatedContentLength int - - for i := 0; i < requestCount; i++ { - w := httptest.NewRecorder() - handler.ServeHTTP(w, req) - - resp := w.Result() - if resp.StatusCode != 200 { - b.Fatalf("expected 200 status code but got %v", resp.StatusCode) - } - - b.StopTimer() - buf := bytes.Buffer{} - buf.ReadFrom(resp.Body) - accumulatedContentLength += buf.Len() - b.StartTimer() - } - - b.SetBytes(int64(accumulatedContentLength)) - }) -} - -// TestFullScrapeCycle is a simple smoke test covering the entire cycle from -// cache filling to scraping. -func TestFullScrapeCycle(t *testing.T) { - t.Parallel() - - kubeClient := fake.NewSimpleClientset() - - err := pod(kubeClient, 0) - if err != nil { - t.Fatalf("failed to insert sample pod %v", err.Error()) - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - reg := prometheus.NewRegistry() - builder := store.NewBuilder() - builder.WithMetrics(reg) - builder.WithEnabledResources(options.DefaultCollectors.AsSlice()) - builder.WithKubeClient(kubeClient) - builder.WithNamespaces(options.DefaultNamespaces) - - l, err := whiteblacklist.New(map[string]struct{}{}, map[string]struct{}{}) - if err != nil { - t.Fatal(err) - } - builder.WithWhiteBlackList(l) - - handler := metricshandler.New(&options.Options{}, kubeClient, builder, false) - handler.ConfigureSharding(ctx, 0, 1) - - // Wait for caches to fill - time.Sleep(time.Second) - - req := httptest.NewRequest("GET", "http://localhost:8080/metrics", nil) - - w := httptest.NewRecorder() - handler.ServeHTTP(w, req) - - resp := w.Result() - if resp.StatusCode != 200 { - t.Fatalf("expected 200 status code but got %v", resp.StatusCode) - } - - body, _ := ioutil.ReadAll(resp.Body) - - expected := `# HELP kube_pod_info Information about pod. -# TYPE kube_pod_info gauge -kube_pod_info{namespace="default",pod="pod0",host_ip="1.1.1.1",pod_ip="1.2.3.4",uid="abc-0",node="node1",created_by_kind="",created_by_name="",priority_class=""} 1 -# HELP kube_pod_start_time Start time in unix timestamp for a pod. -# TYPE kube_pod_start_time gauge -# HELP kube_pod_completion_time Completion time in unix timestamp for a pod. -# TYPE kube_pod_completion_time gauge -# HELP kube_pod_owner Information about the Pod's owner. -# TYPE kube_pod_owner gauge -kube_pod_owner{namespace="default",pod="pod0",owner_kind="",owner_name="",owner_is_controller=""} 1 -# HELP kube_pod_labels Kubernetes labels converted to Prometheus labels. -# TYPE kube_pod_labels gauge -kube_pod_labels{namespace="default",pod="pod0"} 1 -# HELP kube_pod_created Unix creation timestamp -# TYPE kube_pod_created gauge -kube_pod_created{namespace="default",pod="pod0"} 1.5e+09 -# HELP kube_pod_restart_policy Describes the restart policy in use by this pod. -# TYPE kube_pod_restart_policy gauge -kube_pod_restart_policy{namespace="default",pod="pod0",type="Always"} 1 -# HELP kube_pod_status_scheduled_time Unix timestamp when pod moved into scheduled status -# TYPE kube_pod_status_scheduled_time gauge -# HELP kube_pod_status_phase The pods current phase. -# TYPE kube_pod_status_phase gauge -# HELP kube_pod_status_unschedulable Describes the unschedulable status for the pod. -# TYPE kube_pod_status_unschedulable gauge -kube_pod_status_phase{namespace="default",pod="pod0",phase="Pending"} 0 -kube_pod_status_phase{namespace="default",pod="pod0",phase="Succeeded"} 0 -kube_pod_status_phase{namespace="default",pod="pod0",phase="Failed"} 0 -kube_pod_status_phase{namespace="default",pod="pod0",phase="Running"} 1 -kube_pod_status_phase{namespace="default",pod="pod0",phase="Unknown"} 0 -# HELP kube_pod_status_ready Describes whether the pod is ready to serve requests. -# TYPE kube_pod_status_ready gauge -# HELP kube_pod_status_scheduled Describes the status of the scheduling process for the pod. -# TYPE kube_pod_status_scheduled gauge -# HELP kube_pod_container_info Information about a container in a pod. -# TYPE kube_pod_container_info gauge -kube_pod_container_info{namespace="default",pod="pod0",container="container2",image="k8s.gcr.io/hyperkube2",image_id="docker://sha256:bbb",container_id="docker://cd456"} 1 -kube_pod_container_info{namespace="default",pod="pod0",container="container3",image="k8s.gcr.io/hyperkube3",image_id="docker://sha256:ccc",container_id="docker://ef789"} 1 -# HELP kube_pod_init_container_info Information about an init container in a pod. -# TYPE kube_pod_init_container_info gauge -# HELP kube_pod_container_status_waiting Describes whether the container is currently in waiting state. -# TYPE kube_pod_container_status_waiting gauge -kube_pod_container_status_waiting{namespace="default",pod="pod0",container="container2"} 1 -kube_pod_container_status_waiting{namespace="default",pod="pod0",container="container3"} 0 -# HELP kube_pod_init_container_status_waiting Describes whether the init container is currently in waiting state. -# TYPE kube_pod_init_container_status_waiting gauge -# HELP kube_pod_container_status_waiting_reason Describes the reason the container is currently in waiting state. -# TYPE kube_pod_container_status_waiting_reason gauge -kube_pod_container_status_waiting_reason{namespace="default",pod="pod0",container="container2",reason="ContainerCreating"} 0 -kube_pod_container_status_waiting_reason{namespace="default",pod="pod0",container="container2",reason="CrashLoopBackOff"} 1 -kube_pod_container_status_waiting_reason{namespace="default",pod="pod0",container="container2",reason="CreateContainerConfigError"} 0 -kube_pod_container_status_waiting_reason{namespace="default",pod="pod0",container="container2",reason="ErrImagePull"} 0 -kube_pod_container_status_waiting_reason{namespace="default",pod="pod0",container="container2",reason="ImagePullBackOff"} 0 -kube_pod_container_status_waiting_reason{namespace="default",pod="pod0",container="container2",reason="CreateContainerError"} 0 -kube_pod_container_status_waiting_reason{namespace="default",pod="pod0",container="container2",reason="InvalidImageName"} 0 -kube_pod_container_status_waiting_reason{namespace="default",pod="pod0",container="container3",reason="ContainerCreating"} 0 -kube_pod_container_status_waiting_reason{namespace="default",pod="pod0",container="container3",reason="CrashLoopBackOff"} 0 -kube_pod_container_status_waiting_reason{namespace="default",pod="pod0",container="container3",reason="CreateContainerConfigError"} 0 -kube_pod_container_status_waiting_reason{namespace="default",pod="pod0",container="container3",reason="ErrImagePull"} 0 -kube_pod_container_status_waiting_reason{namespace="default",pod="pod0",container="container3",reason="ImagePullBackOff"} 0 -kube_pod_container_status_waiting_reason{namespace="default",pod="pod0",container="container3",reason="CreateContainerError"} 0 -kube_pod_container_status_waiting_reason{namespace="default",pod="pod0",container="container3",reason="InvalidImageName"} 0 -# HELP kube_pod_init_container_status_waiting_reason Describes the reason the init container is currently in waiting state. -# TYPE kube_pod_init_container_status_waiting_reason gauge -# HELP kube_pod_container_status_running Describes whether the container is currently in running state. -# TYPE kube_pod_container_status_running gauge -kube_pod_container_status_running{namespace="default",pod="pod0",container="container2"} 0 -kube_pod_container_status_running{namespace="default",pod="pod0",container="container3"} 0 -# HELP kube_pod_init_container_status_running Describes whether the init container is currently in running state. -# TYPE kube_pod_init_container_status_running gauge -# HELP kube_pod_container_status_terminated Describes whether the container is currently in terminated state. -# TYPE kube_pod_container_status_terminated gauge -kube_pod_container_status_terminated{namespace="default",pod="pod0",container="container2"} 0 -kube_pod_container_status_terminated{namespace="default",pod="pod0",container="container3"} 0 -# HELP kube_pod_init_container_status_terminated Describes whether the init container is currently in terminated state. -# TYPE kube_pod_init_container_status_terminated gauge -# HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state. -# TYPE kube_pod_container_status_terminated_reason gauge -kube_pod_container_status_terminated_reason{namespace="default",pod="pod0",container="container2",reason="OOMKilled"} 0 -kube_pod_container_status_terminated_reason{namespace="default",pod="pod0",container="container2",reason="Completed"} 0 -kube_pod_container_status_terminated_reason{namespace="default",pod="pod0",container="container2",reason="Error"} 0 -kube_pod_container_status_terminated_reason{namespace="default",pod="pod0",container="container2",reason="Evicted"} 0 -kube_pod_container_status_terminated_reason{namespace="default",pod="pod0",container="container2",reason="ContainerCannotRun"} 0 -kube_pod_container_status_terminated_reason{namespace="default",pod="pod0",container="container2",reason="DeadlineExceeded"} 0 -kube_pod_container_status_terminated_reason{namespace="default",pod="pod0",container="container3",reason="OOMKilled"} 0 -kube_pod_container_status_terminated_reason{namespace="default",pod="pod0",container="container3",reason="Completed"} 0 -kube_pod_container_status_terminated_reason{namespace="default",pod="pod0",container="container3",reason="Error"} 0 -kube_pod_container_status_terminated_reason{namespace="default",pod="pod0",container="container3",reason="Evicted"} 0 -kube_pod_container_status_terminated_reason{namespace="default",pod="pod0",container="container3",reason="ContainerCannotRun"} 0 -kube_pod_container_status_terminated_reason{namespace="default",pod="pod0",container="container3",reason="DeadlineExceeded"} 0 -# HELP kube_pod_init_container_status_terminated_reason Describes the reason the init container is currently in terminated state. -# TYPE kube_pod_init_container_status_terminated_reason gauge -# HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state. -# TYPE kube_pod_container_status_last_terminated_reason gauge -kube_pod_container_status_last_terminated_reason{namespace="default",pod="pod0",container="container2",reason="OOMKilled"} 1 -kube_pod_container_status_last_terminated_reason{namespace="default",pod="pod0",container="container2",reason="Completed"} 0 -kube_pod_container_status_last_terminated_reason{namespace="default",pod="pod0",container="container2",reason="Error"} 0 -kube_pod_container_status_last_terminated_reason{namespace="default",pod="pod0",container="container2",reason="Evicted"} 0 -kube_pod_container_status_last_terminated_reason{namespace="default",pod="pod0",container="container2",reason="ContainerCannotRun"} 0 -kube_pod_container_status_last_terminated_reason{namespace="default",pod="pod0",container="container2",reason="DeadlineExceeded"} 0 -kube_pod_container_status_last_terminated_reason{namespace="default",pod="pod0",container="container3",reason="OOMKilled"} 0 -kube_pod_container_status_last_terminated_reason{namespace="default",pod="pod0",container="container3",reason="Completed"} 0 -kube_pod_container_status_last_terminated_reason{namespace="default",pod="pod0",container="container3",reason="Error"} 0 -kube_pod_container_status_last_terminated_reason{namespace="default",pod="pod0",container="container3",reason="Evicted"} 0 -kube_pod_container_status_last_terminated_reason{namespace="default",pod="pod0",container="container3",reason="ContainerCannotRun"} 0 -kube_pod_container_status_last_terminated_reason{namespace="default",pod="pod0",container="container3",reason="DeadlineExceeded"} 0 -# HELP kube_pod_init_container_status_last_terminated_reason Describes the last reason the init container was in terminated state. -# TYPE kube_pod_init_container_status_last_terminated_reason gauge -# HELP kube_pod_container_status_ready Describes whether the containers readiness check succeeded. -# TYPE kube_pod_container_status_ready gauge -kube_pod_container_status_ready{namespace="default",pod="pod0",container="container2"} 0 -kube_pod_container_status_ready{namespace="default",pod="pod0",container="container3"} 0 -# HELP kube_pod_init_container_status_ready Describes whether the init containers readiness check succeeded. -# TYPE kube_pod_init_container_status_ready gauge -# HELP kube_pod_container_status_restarts_total The number of container restarts per container. -# TYPE kube_pod_container_status_restarts_total counter -kube_pod_container_status_restarts_total{namespace="default",pod="pod0",container="container2"} 0 -kube_pod_container_status_restarts_total{namespace="default",pod="pod0",container="container3"} 0 -# HELP kube_pod_init_container_status_restarts_total The number of restarts for the init container. -# TYPE kube_pod_init_container_status_restarts_total counter -# HELP kube_pod_container_resource_requests The number of requested request resource by a container. -# TYPE kube_pod_container_resource_requests gauge -kube_pod_container_resource_requests{namespace="default",pod="pod0",container="pod1_con1",node="node1",resource="nvidia_com_gpu",unit="integer"} 1 -kube_pod_container_resource_requests{namespace="default",pod="pod0",container="pod1_con1",node="node1",resource="cpu",unit="core"} 0.2 -kube_pod_container_resource_requests{namespace="default",pod="pod0",container="pod1_con1",node="node1",resource="memory",unit="byte"} 1e+08 -kube_pod_container_resource_requests{namespace="default",pod="pod0",container="pod1_con1",node="node1",resource="ephemeral_storage",unit="byte"} 3e+08 -kube_pod_container_resource_requests{namespace="default",pod="pod0",container="pod1_con1",node="node1",resource="storage",unit="byte"} 4e+08 -kube_pod_container_resource_requests{namespace="default",pod="pod0",container="pod1_con2",node="node1",resource="cpu",unit="core"} 0.3 -kube_pod_container_resource_requests{namespace="default",pod="pod0",container="pod1_con2",node="node1",resource="memory",unit="byte"} 2e+08 -# HELP kube_pod_init_container_resource_limits The number of requested limit resource by the init container. -# TYPE kube_pod_init_container_resource_limits gauge -# HELP kube_pod_container_resource_limits The number of requested limit resource by a container. -# TYPE kube_pod_container_resource_limits gauge -kube_pod_container_resource_limits{namespace="default",pod="pod0",container="pod1_con1",node="node1",resource="nvidia_com_gpu",unit="integer"} 1 -kube_pod_container_resource_limits{namespace="default",pod="pod0",container="pod1_con1",node="node1",resource="cpu",unit="core"} 0.2 -kube_pod_container_resource_limits{namespace="default",pod="pod0",container="pod1_con1",node="node1",resource="memory",unit="byte"} 1e+08 -kube_pod_container_resource_limits{namespace="default",pod="pod0",container="pod1_con1",node="node1",resource="ephemeral_storage",unit="byte"} 3e+08 -kube_pod_container_resource_limits{namespace="default",pod="pod0",container="pod1_con1",node="node1",resource="storage",unit="byte"} 4e+08 -kube_pod_container_resource_limits{namespace="default",pod="pod0",container="pod1_con2",node="node1",resource="memory",unit="byte"} 2e+08 -kube_pod_container_resource_limits{namespace="default",pod="pod0",container="pod1_con2",node="node1",resource="cpu",unit="core"} 0.3 -# HELP kube_pod_container_resource_requests_cpu_cores The number of requested cpu cores by a container. -# TYPE kube_pod_container_resource_requests_cpu_cores gauge -kube_pod_container_resource_requests_cpu_cores{namespace="default",pod="pod0",container="pod1_con1",node="node1"} 0.2 -kube_pod_container_resource_requests_cpu_cores{namespace="default",pod="pod0",container="pod1_con2",node="node1"} 0.3 -# HELP kube_pod_container_resource_requests_memory_bytes The number of requested memory bytes by a container. -# TYPE kube_pod_container_resource_requests_memory_bytes gauge -kube_pod_container_resource_requests_memory_bytes{namespace="default",pod="pod0",container="pod1_con1",node="node1"} 1e+08 -kube_pod_container_resource_requests_memory_bytes{namespace="default",pod="pod0",container="pod1_con2",node="node1"} 2e+08 -# HELP kube_pod_container_resource_limits_cpu_cores The limit on cpu cores to be used by a container. -# TYPE kube_pod_container_resource_limits_cpu_cores gauge -kube_pod_container_resource_limits_cpu_cores{namespace="default",pod="pod0",container="pod1_con1",node="node1"} 0.2 -kube_pod_container_resource_limits_cpu_cores{namespace="default",pod="pod0",container="pod1_con2",node="node1"} 0.3 -# HELP kube_pod_container_resource_limits_memory_bytes The limit on memory to be used by a container in bytes. -# TYPE kube_pod_container_resource_limits_memory_bytes gauge -kube_pod_container_resource_limits_memory_bytes{namespace="default",pod="pod0",container="pod1_con1",node="node1"} 1e+08 -kube_pod_container_resource_limits_memory_bytes{namespace="default",pod="pod0",container="pod1_con2",node="node1"} 2e+08 -# HELP kube_pod_spec_volumes_persistentvolumeclaims_info Information about persistentvolumeclaim volumes in a pod. -# TYPE kube_pod_spec_volumes_persistentvolumeclaims_info gauge -# HELP kube_pod_spec_volumes_persistentvolumeclaims_readonly Describes whether a persistentvolumeclaim is mounted read only. -# TYPE kube_pod_spec_volumes_persistentvolumeclaims_readonly gauge` - - expectedSplit := strings.Split(strings.TrimSpace(expected), "\n") - sort.Strings(expectedSplit) - - gotSplit := strings.Split(strings.TrimSpace(string(body)), "\n") - - gotFiltered := []string{} - for _, l := range gotSplit { - if strings.Contains(l, "kube_pod_") { - gotFiltered = append(gotFiltered, l) - } - } - - sort.Strings(gotFiltered) - - if len(expectedSplit) != len(gotFiltered) { - t.Fatalf("expected different output length, expected %d got %d", len(expectedSplit), len(gotFiltered)) - } - - for i := 0; i < len(expectedSplit); i++ { - if expectedSplit[i] != gotFiltered[i] { - t.Fatalf("expected:\n\n%v, but got:\n\n%v", expectedSplit[i], gotFiltered[i]) - } - } -} - -// TestShardingEquivalenceScrapeCycle is a simple smoke test covering the entire cycle from -// cache filling to scraping comparing a sharded with an unsharded setup. -func TestShardingEquivalenceScrapeCycle(t *testing.T) { - t.Parallel() - - kubeClient := fake.NewSimpleClientset() - - for i := 0; i < 10; i++ { - err := pod(kubeClient, i) - if err != nil { - t.Fatalf("failed to insert sample pod %v", err.Error()) - } - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - l, err := whiteblacklist.New(map[string]struct{}{}, map[string]struct{}{}) - if err != nil { - t.Fatal(err) - } - - reg := prometheus.NewRegistry() - unshardedBuilder := store.NewBuilder() - unshardedBuilder.WithMetrics(reg) - unshardedBuilder.WithEnabledResources(options.DefaultCollectors.AsSlice()) - unshardedBuilder.WithKubeClient(kubeClient) - unshardedBuilder.WithNamespaces(options.DefaultNamespaces) - unshardedBuilder.WithWhiteBlackList(l) - - unshardedHandler := metricshandler.New(&options.Options{}, kubeClient, unshardedBuilder, false) - unshardedHandler.ConfigureSharding(ctx, 0, 1) - - regShard1 := prometheus.NewRegistry() - shardedBuilder1 := store.NewBuilder() - shardedBuilder1.WithMetrics(regShard1) - shardedBuilder1.WithEnabledResources(options.DefaultCollectors.AsSlice()) - shardedBuilder1.WithKubeClient(kubeClient) - shardedBuilder1.WithNamespaces(options.DefaultNamespaces) - shardedBuilder1.WithWhiteBlackList(l) - - shardedHandler1 := metricshandler.New(&options.Options{}, kubeClient, shardedBuilder1, false) - shardedHandler1.ConfigureSharding(ctx, 0, 2) - - regShard2 := prometheus.NewRegistry() - shardedBuilder2 := store.NewBuilder() - shardedBuilder2.WithMetrics(regShard2) - shardedBuilder2.WithEnabledResources(options.DefaultCollectors.AsSlice()) - shardedBuilder2.WithKubeClient(kubeClient) - shardedBuilder2.WithNamespaces(options.DefaultNamespaces) - shardedBuilder2.WithWhiteBlackList(l) - - shardedHandler2 := metricshandler.New(&options.Options{}, kubeClient, shardedBuilder2, false) - shardedHandler2.ConfigureSharding(ctx, 1, 2) - - // Wait for caches to fill - time.Sleep(time.Second) - - // unsharded request as the controlled environment - req := httptest.NewRequest("GET", "http://localhost:8080/metrics", nil) - - w := httptest.NewRecorder() - unshardedHandler.ServeHTTP(w, req) - - resp := w.Result() - if resp.StatusCode != 200 { - t.Fatalf("expected 200 status code but got %v", resp.StatusCode) - } - - body, _ := ioutil.ReadAll(resp.Body) - expected := string(body) - - // sharded requests - // - // request first shard - req = httptest.NewRequest("GET", "http://localhost:8080/metrics", nil) - - w = httptest.NewRecorder() - shardedHandler1.ServeHTTP(w, req) - - resp = w.Result() - if resp.StatusCode != 200 { - t.Fatalf("expected 200 status code but got %v", resp.StatusCode) - } - - body, _ = ioutil.ReadAll(resp.Body) - got1 := string(body) - - // request second shard - req = httptest.NewRequest("GET", "http://localhost:8080/metrics", nil) - - w = httptest.NewRecorder() - shardedHandler2.ServeHTTP(w, req) - - resp = w.Result() - if resp.StatusCode != 200 { - t.Fatalf("expected 200 status code but got %v", resp.StatusCode) - } - - body, _ = ioutil.ReadAll(resp.Body) - got2 := string(body) - - // normalize results: - - expectedSplit := strings.Split(strings.TrimSpace(expected), "\n") - sort.Strings(expectedSplit) - - expectedFiltered := []string{} - for _, l := range expectedSplit { - if strings.HasPrefix(l, "kube_pod_") { - expectedFiltered = append(expectedFiltered, l) - } - } - - got1Split := strings.Split(strings.TrimSpace(got1), "\n") - sort.Strings(got1Split) - - got1Filtered := []string{} - for _, l := range got1Split { - if strings.HasPrefix(l, "kube_pod_") { - got1Filtered = append(got1Filtered, l) - } - } - - got2Split := strings.Split(strings.TrimSpace(got2), "\n") - sort.Strings(got2Split) - - got2Filtered := []string{} - for _, l := range got2Split { - if strings.HasPrefix(l, "kube_pod_") { - got2Filtered = append(got2Filtered, l) - } - } - - // total metrics should be equal - if len(expectedFiltered) != (len(got1Filtered) + len(got2Filtered)) { - t.Fatalf("expected different output length, expected total %d got 1) %d 2) %d", len(expectedFiltered), len(got1Filtered), len(got2Filtered)) - } - // smoke test to test that each shard actually represents a subset - if len(got1Filtered) == 0 { - t.Fatal("shard 1 has 0 metrics when it shouldn't") - } - if len(got2Filtered) == 0 { - t.Fatal("shard 2 has 0 metrics when it shouldn't") - } - - gotFiltered := append(got1Filtered, got2Filtered...) - sort.Strings(gotFiltered) - - for i := 0; i < len(expectedFiltered); i++ { - expected := strings.TrimSpace(expectedFiltered[i]) - got := strings.TrimSpace(gotFiltered[i]) - if expected != got { - t.Fatalf("\n\nexpected:\n\n%q\n\nbut got:\n\n%q\n\n", expected, got) - } - } -} - -func injectFixtures(client *fake.Clientset, multiplier int) error { - creators := []func(*fake.Clientset, int) error{ - configMap, - service, - pod, - } - - for _, c := range creators { - for i := 0; i < multiplier; i++ { - err := c(client, i) - - if err != nil { - return err - } - } - } - - return nil -} - -func configMap(client *fake.Clientset, index int) error { - i := strconv.Itoa(index) - - configMap := v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "configmap" + i, - ResourceVersion: "123456", - UID: types.UID("abc-" + i), - }, - } - _, err := client.CoreV1().ConfigMaps(metav1.NamespaceDefault).Create(&configMap) - return err -} - -func service(client *fake.Clientset, index int) error { - i := strconv.Itoa(index) - - service := v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "service" + i, - ResourceVersion: "123456", - UID: types.UID("abc-" + i), - }, - } - _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(&service) - return err -} - -func pod(client *fake.Clientset, index int) error { - i := strconv.Itoa(index) - - pod := v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod" + i, - CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, - Namespace: "default", - UID: types.UID("abc-" + i), - }, - Spec: v1.PodSpec{ - RestartPolicy: v1.RestartPolicyAlways, - NodeName: "node1", - Containers: []v1.Container{ - { - Name: "pod1_con1", - Resources: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("200m"), - v1.ResourceMemory: resource.MustParse("100M"), - v1.ResourceEphemeralStorage: resource.MustParse("300M"), - v1.ResourceStorage: resource.MustParse("400M"), - v1.ResourceName("nvidia.com/gpu"): resource.MustParse("1"), - }, - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("200m"), - v1.ResourceMemory: resource.MustParse("100M"), - v1.ResourceEphemeralStorage: resource.MustParse("300M"), - v1.ResourceStorage: resource.MustParse("400M"), - v1.ResourceName("nvidia.com/gpu"): resource.MustParse("1"), - }, - }, - }, - { - Name: "pod1_con2", - Resources: v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("300m"), - v1.ResourceMemory: resource.MustParse("200M"), - }, - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("300m"), - v1.ResourceMemory: resource.MustParse("200M"), - }, - }, - }, - }, - }, - Status: v1.PodStatus{ - HostIP: "1.1.1.1", - PodIP: "1.2.3.4", - Phase: v1.PodRunning, - ContainerStatuses: []v1.ContainerStatus{ - { - Name: "container2", - Image: "k8s.gcr.io/hyperkube2", - ImageID: "docker://sha256:bbb", - ContainerID: "docker://cd456", - State: v1.ContainerState{ - Waiting: &v1.ContainerStateWaiting{ - Reason: "CrashLoopBackOff", - }, - }, - LastTerminationState: v1.ContainerState{ - Terminated: &v1.ContainerStateTerminated{ - Reason: "OOMKilled", - }, - }, - }, - { - Name: "container3", - Image: "k8s.gcr.io/hyperkube3", - ImageID: "docker://sha256:ccc", - ContainerID: "docker://ef789", - }, - }, - }, - } - - _, err := client.CoreV1().Pods(metav1.NamespaceDefault).Create(&pod) - return err -} diff --git a/noops.travis.yml b/noops.travis.yml deleted file mode 100644 index 6427747567..0000000000 --- a/noops.travis.yml +++ /dev/null @@ -1,50 +0,0 @@ -sudo: required -dist: xenial -language: go - -go: - - "1.13.x" - -services: - - docker - -git: - # Benchmark tests needs to access other branches, thereby we need to fetch all - # of the repository. - depth: false - -env: - global: - - E2E_SETUP_MINIKUBE=yes - - E2E_SETUP_KUBECTL=yes - - E2E_SETUP_PROMTOOL=yes - - MINIKUBE_DRIVER=none - - SUDO=sudo - -before_script: -- make install-tools - -install: -- mkdir -p $HOME/gopath/src/k8s.io -- mv $TRAVIS_BUILD_DIR $HOME/gopath/src/k8s.io/kube-state-metrics -- cd $HOME/gopath/src/k8s.io/kube-state-metrics - -jobs: - include: - - stage: all - name: Lint - script: make lint - - name: Validate generated manifests - script: make validate-manifests - - name: Validate vendor is in sync with go modules - script: make validate-modules - - name: Check that all metrics are documented - script: make doccheck - - name: Unit tests - script: make test-unit - - name: Benchmark tests - script: make test-benchmark-compare - - name: Build - script: make build - - name: End to end tests - script: make e2e diff --git a/pkg/allow/allow_labels.go b/pkg/allow/allow_labels.go new file mode 100644 index 0000000000..6051d14fb4 --- /dev/null +++ b/pkg/allow/allow_labels.go @@ -0,0 +1,76 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package allow + +import ( + "regexp" +) + +var defaultMetricLabels = map[*regexp.Regexp][]string{ + regexp.MustCompile(".*_labels$"): {}, + regexp.MustCompile(".*_annotations$"): {}, +} + +// Labels provide a way to allow only specified labels for metrics. +// Falls back to default labels, if your metric doesn't have defaults +// then all labels are allowed. +type Labels map[string][]string + +// Allowed returns allowed metric keys and values for the metric. +func (a Labels) Allowed(metric string, labels, values []string) ([]string, []string) { + allowedLabels, ok := a[metric] + if !ok { + var defaultsPresent bool + for metricReg, lbls := range defaultMetricLabels { + if metricReg.MatchString(metric) { + allowedLabels = append(allowedLabels, lbls...) + defaultsPresent = true + } + } + + if !defaultsPresent { + return labels, values + } + } + + var finalLabels, finalValues []string + labelSet := labelSet(allowedLabels) + for _, allowedLabel := range labelSet { + for i, label := range labels { + if label == allowedLabel { + finalLabels = append(finalLabels, label) + finalValues = append(finalValues, values[i]) + } + } + } + + return finalLabels, finalValues +} + +func labelSet(lists ...[]string) []string { + m := make(map[string]interface{}) + var set []string + for _, list := range lists { + for _, e := range list { + if _, ok := m[e]; !ok { + m[e] = struct{}{} + set = append(set, e) + } + } + } + return set +} diff --git a/pkg/whiteblacklist/whiteblacklist.go b/pkg/allowdenylist/allowdenylist.go similarity index 54% rename from pkg/whiteblacklist/whiteblacklist.go rename to pkg/allowdenylist/allowdenylist.go index 8dd1a9c8d7..bf6a427d53 100644 --- a/pkg/whiteblacklist/whiteblacklist.go +++ b/pkg/allowdenylist/allowdenylist.go @@ -14,51 +14,52 @@ See the License for the specific language governing permissions and limitations under the License. */ -package whiteblacklist +package allowdenylist import ( + "errors" "regexp" "strings" - "github.com/pkg/errors" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) -// WhiteBlackList encapsulates the logic needed to filter based on a string. -type WhiteBlackList struct { +// AllowDenyList encapsulates the logic needed to filter based on a string. +type AllowDenyList struct { list map[string]struct{} rList []*regexp.Regexp - isWhiteList bool + isAllowList bool } -// New constructs a new WhiteBlackList based on a white- and a -// blacklist. Only one of them can be not empty. -func New(white, black map[string]struct{}) (*WhiteBlackList, error) { - if len(white) != 0 && len(black) != 0 { +// New constructs a new AllowDenyList based on a allow- and a +// denylist. Only one of them can be not empty. +func New(allow, deny map[string]struct{}) (*AllowDenyList, error) { + if len(allow) != 0 && len(deny) != 0 { return nil, errors.New( - "whitelist and blacklist are both set, they are mutually exclusive, only one of them can be set", + "allowlist and denylist are both set, they are mutually exclusive, only one of them can be set", ) } var list map[string]struct{} - var isWhiteList bool + var isAllowList bool - // Default to blacklisting - if len(white) != 0 { - list = copyList(white) - isWhiteList = true + // Default to denylisting + if len(allow) != 0 { + list = copyList(allow) + isAllowList = true } else { - list = copyList(black) - isWhiteList = false + list = copyList(deny) + isAllowList = false } - return &WhiteBlackList{ + return &AllowDenyList{ list: list, - isWhiteList: isWhiteList, + isAllowList: isAllowList, }, nil } -// Parse parses and compiles all of the regexes in the whiteBlackList. -func (l *WhiteBlackList) Parse() error { +// Parse parses and compiles all of the regexes in the allowDenyList. +func (l *AllowDenyList) Parse() error { regexes := make([]*regexp.Regexp, 0, len(l.list)) for item := range l.list { r, err := regexp.Compile(item) @@ -72,8 +73,8 @@ func (l *WhiteBlackList) Parse() error { } // Include includes the given items in the list. -func (l *WhiteBlackList) Include(items []string) { - if l.isWhiteList { +func (l *AllowDenyList) Include(items []string) { + if l.isAllowList { for _, item := range items { l.list[item] = struct{}{} } @@ -85,8 +86,8 @@ func (l *WhiteBlackList) Include(items []string) { } // Exclude excludes the given items from the list. -func (l *WhiteBlackList) Exclude(items []string) { - if l.isWhiteList { +func (l *AllowDenyList) Exclude(items []string) { + if l.isAllowList { for _, item := range items { delete(l.list, item) } @@ -98,7 +99,7 @@ func (l *WhiteBlackList) Exclude(items []string) { } // IsIncluded returns if the given item is included. -func (l *WhiteBlackList) IsIncluded(item string) bool { +func (l *AllowDenyList) IsIncluded(item string) bool { var matched bool for _, r := range l.rList { matched = r.MatchString(item) @@ -107,7 +108,7 @@ func (l *WhiteBlackList) IsIncluded(item string) bool { } } - if l.isWhiteList { + if l.isAllowList { return matched } @@ -115,23 +116,28 @@ func (l *WhiteBlackList) IsIncluded(item string) bool { } // IsExcluded returns if the given item is excluded. -func (l *WhiteBlackList) IsExcluded(item string) bool { +func (l *AllowDenyList) IsExcluded(item string) bool { return !l.IsIncluded(item) } -// Status returns the status of the WhiteBlackList that can e.g. be passed into +// Status returns the status of the AllowDenyList that can e.g. be passed into // a logger. -func (l *WhiteBlackList) Status() string { +func (l *AllowDenyList) Status() string { items := make([]string, 0, len(l.list)) for key := range l.list { items = append(items, key) } - if l.isWhiteList { - return "whitelisting the following items: " + strings.Join(items, ", ") + if l.isAllowList { + return "Including the following lists that were on allowlist: " + strings.Join(items, ", ") } - return "blacklisting the following items: " + strings.Join(items, ", ") + return "Excluding the following lists that were on denylist: " + strings.Join(items, ", ") +} + +// Test returns if the given family generator passes (is included in) the AllowDenyList +func (l *AllowDenyList) Test(generator generator.FamilyGenerator) bool { + return l.IsIncluded(generator.Name) } func copyList(l map[string]struct{}) map[string]struct{} { diff --git a/pkg/whiteblacklist/whiteblacklist_test.go b/pkg/allowdenylist/allowdenylist_test.go similarity index 57% rename from pkg/whiteblacklist/whiteblacklist_test.go rename to pkg/allowdenylist/allowdenylist_test.go index 0c78c77f1f..1199dc1521 100644 --- a/pkg/whiteblacklist/whiteblacklist_test.go +++ b/pkg/allowdenylist/allowdenylist_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package whiteblacklist +package allowdenylist import ( "regexp" @@ -29,154 +29,154 @@ func TestNew(t *testing.T) { } }) - t.Run("defaults to blacklisting", func(t *testing.T) { + t.Run("defaults to denylisting", func(t *testing.T) { l, err := New(map[string]struct{}{}, map[string]struct{}{}) if err != nil { t.Fatal("expected New() to not fail") } - if l.isWhiteList { - t.Fatal("expected whiteBlackList to default to blacklist") + if l.isAllowList { + t.Fatal("expected allowDenyList to default to denylist") } }) - t.Run("if whitelist set, should be whitelist", func(t *testing.T) { + t.Run("if allowlist set, should be allowlist", func(t *testing.T) { list, err := New(map[string]struct{}{"not-empty": {}}, map[string]struct{}{}) if err != nil { t.Fatal("expected New() to not fail") } - if !list.isWhiteList { - t.Fatal("expected list to be whitelist") + if !list.isAllowList { + t.Fatal("expected list to be allowlist") } }) - t.Run("if blacklist set, should be blacklist", func(t *testing.T) { + t.Run("if denylist set, should be denylist", func(t *testing.T) { list, err := New(map[string]struct{}{}, map[string]struct{}{"not-empty": {}}) if err != nil { t.Fatal("expected New() to not fail") } - if list.isWhiteList { - t.Fatal("expected list to be blacklist") + if list.isAllowList { + t.Fatal("expected list to be denylist") } }) } func TestInclude(t *testing.T) { - t.Run("adds when whitelist", func(t *testing.T) { - whitelist, err := New(map[string]struct{}{"not-empty": {}}, map[string]struct{}{}) + t.Run("adds when allowlist", func(t *testing.T) { + allowlist, err := New(map[string]struct{}{"not-empty": {}}, map[string]struct{}{}) if err != nil { t.Fatal("expected New() to not fail") } - whitelist.Include([]string{"item1"}) - err = whitelist.Parse() + allowlist.Include([]string{"item1"}) + err = allowlist.Parse() if err != nil { t.Fatal("expected Parse() to not fail") } - if !whitelist.IsIncluded("item1") { + if !allowlist.IsIncluded("item1") { t.Fatal("expected included item to be included") } }) - t.Run("removes when blacklist", func(t *testing.T) { + t.Run("removes when denylist", func(t *testing.T) { item1 := "item1" - blacklist, err := New(map[string]struct{}{}, map[string]struct{}{item1: {}}) + denylist, err := New(map[string]struct{}{}, map[string]struct{}{item1: {}}) if err != nil { t.Fatal("expected New() to not fail") } - blacklist.Include([]string{item1}) - err = blacklist.Parse() + denylist.Include([]string{item1}) + err = denylist.Parse() if err != nil { t.Fatalf("expected Parse() to not fail, but got error : %v", err) } - if !blacklist.IsIncluded(item1) { + if !denylist.IsIncluded(item1) { t.Fatal("expected included item to be included") } }) - t.Run("adds during pattern match when in whitelist mode", func(t *testing.T) { - whitelist, err := New(map[string]struct{}{"not-empty": {}}, map[string]struct{}{}) + t.Run("adds during pattern match when in allowlist mode", func(t *testing.T) { + allowlist, err := New(map[string]struct{}{"not-empty": {}}, map[string]struct{}{}) if err != nil { t.Fatal("expected New() to not fail") } - whitelist.Include([]string{"kube_.*_info"}) - err = whitelist.Parse() + allowlist.Include([]string{"kube_.*_info"}) + err = allowlist.Parse() if err != nil { t.Fatalf("expected Parse() to not fail, but got error : %v", err) } - if !whitelist.IsIncluded("kube_secret_info") { + if !allowlist.IsIncluded("kube_secret_info") { t.Fatal("expected included item to be included") } }) - t.Run("removes during pattern match when in blackist mode", func(t *testing.T) { + t.Run("removes during pattern match when in denyist mode", func(t *testing.T) { item1 := "kube_pod_container_resource_requests_cpu_cores" item2 := "kube_pod_container_resource_requests_memory_bytes" item3 := "kube_node_status_capacity_cpu_cores" item4 := "kube_node_status_capacity_memory_bytes" - blacklist, err := New(map[string]struct{}{}, map[string]struct{}{}) + denylist, err := New(map[string]struct{}{}, map[string]struct{}{}) if err != nil { t.Fatal("expected New() to not fail") } - blacklist.Exclude([]string{"kube_node_.*_cores", "kube_pod_.*_bytes"}) - err = blacklist.Parse() + denylist.Exclude([]string{"kube_node_.*_cores", "kube_pod_.*_bytes"}) + err = denylist.Parse() if err != nil { t.Fatalf("expected Parse() to not fail, but got error : %v", err) } - if blacklist.IsExcluded(item1) { + if denylist.IsExcluded(item1) { t.Fatalf("expected included %s to be included", item1) } - if blacklist.IsIncluded(item2) { + if denylist.IsIncluded(item2) { t.Fatalf("expected included %s to be excluded", item2) } - if blacklist.IsIncluded(item3) { + if denylist.IsIncluded(item3) { t.Fatalf("expected included %s to be excluded", item3) } - if blacklist.IsExcluded(item4) { + if denylist.IsExcluded(item4) { t.Fatalf("expected included %s to be included", item4) } }) } func TestExclude(t *testing.T) { - t.Run("removes when whitelist", func(t *testing.T) { + t.Run("removes when allowlist", func(t *testing.T) { item1 := "item1" - whitelist, err := New(map[string]struct{}{item1: {}}, map[string]struct{}{}) + allowlist, err := New(map[string]struct{}{item1: {}}, map[string]struct{}{}) if err != nil { t.Fatal("expected New() to not fail") } - whitelist.Exclude([]string{item1}) - err = whitelist.Parse() + allowlist.Exclude([]string{item1}) + err = allowlist.Parse() if err != nil { t.Fatalf("expected Parse() to not fail, but got error : %v", err) } - if whitelist.IsIncluded(item1) { + if allowlist.IsIncluded(item1) { t.Fatal("expected excluded item to be excluded") } }) - t.Run("removes when blacklist", func(t *testing.T) { + t.Run("removes when denylist", func(t *testing.T) { item1 := "item1" - blacklist, err := New(map[string]struct{}{}, map[string]struct{}{"not-empty": {}}) + denylist, err := New(map[string]struct{}{}, map[string]struct{}{"not-empty": {}}) if err != nil { t.Fatal("expected New() to not fail") } - blacklist.Exclude([]string{item1}) - err = blacklist.Parse() + denylist.Exclude([]string{item1}) + err = denylist.Parse() if err != nil { t.Fatalf("expected Parse() to not fail, but got error : %v", err) } - if blacklist.IsIncluded(item1) { + if denylist.IsIncluded(item1) { t.Fatal("expected excluded item to be excluded") } }) @@ -187,7 +187,7 @@ func TestParse(t *testing.T) { invalidItem := "*_pod_info" wb, err := New(map[string]struct{}{invalidItem: {}}, map[string]struct{}{}) if err != nil { - t.Fatalf("unexpected error while trying to init a whiteBlackList: %v", wb) + t.Fatalf("unexpected error while trying to init a allowDenyList: %v", wb) } err = wb.Parse() if err == nil { @@ -195,55 +195,55 @@ func TestParse(t *testing.T) { } }) - t.Run("parses successfully when the whiteBlackList has valid regexes", func(t *testing.T) { + t.Run("parses successfully when the allowDenyList has valid regexes", func(t *testing.T) { validItem := "kube_.*_info" wb, err := New(map[string]struct{}{validItem: {}}, map[string]struct{}{}) if err != nil { - t.Fatalf("unexpected error while trying to init a whiteBlackList: %v", wb) + t.Fatalf("unexpected error while trying to init a allowDenyList: %v", wb) } err = wb.Parse() if err != nil { - t.Errorf("unexpected error while attempting to parse whiteBlackList : %v", err) + t.Errorf("unexpected error while attempting to parse allowDenyList : %v", err) } }) } func TestStatus(t *testing.T) { - t.Run("status when whitelist has single item", func(t *testing.T) { + t.Run("status when allowlist has single item", func(t *testing.T) { item1 := "item1" - whitelist, _ := New(map[string]struct{}{item1: {}}, map[string]struct{}{}) - actualStatusString := whitelist.Status() - expectedStatusString := "whitelisting the following items: " + item1 + allowlist, _ := New(map[string]struct{}{item1: {}}, map[string]struct{}{}) + actualStatusString := allowlist.Status() + expectedStatusString := "Including the following lists that were on allowlist: " + item1 if actualStatusString != expectedStatusString { t.Errorf("expected status %q but got %q", expectedStatusString, actualStatusString) } }) - t.Run("status when whitelist has multiple items", func(t *testing.T) { + t.Run("status when allowlist has multiple items", func(t *testing.T) { item1 := "item1" item2 := "item2" - whitelist, _ := New(map[string]struct{}{item1: {}, item2: {}}, map[string]struct{}{}) - actualStatusString := whitelist.Status() - expectedRegexPattern := `^whitelisting the following items: (item1|item2), (item2|item1)$` + allowlist, _ := New(map[string]struct{}{item1: {}, item2: {}}, map[string]struct{}{}) + actualStatusString := allowlist.Status() + expectedRegexPattern := `^Including the following lists that were on allowlist: (item1|item2), (item2|item1)$` matched, _ := regexp.MatchString(expectedRegexPattern, actualStatusString) if !matched { t.Errorf("expected status %q but got %q", expectedRegexPattern, actualStatusString) } }) - t.Run("status when blacklist has single item", func(t *testing.T) { + t.Run("status when denylist has single item", func(t *testing.T) { item1 := "not-empty" - blacklist, _ := New(map[string]struct{}{}, map[string]struct{}{item1: {}}) - actualStatusString := blacklist.Status() - expectedStatusString := "blacklisting the following items: " + item1 + denylist, _ := New(map[string]struct{}{}, map[string]struct{}{item1: {}}) + actualStatusString := denylist.Status() + expectedStatusString := "Excluding the following lists that were on denylist: " + item1 if actualStatusString != expectedStatusString { t.Errorf("expected status %q but got %q", expectedStatusString, actualStatusString) } }) - t.Run("status when blacklist has multiple items", func(t *testing.T) { + t.Run("status when denylist has multiple items", func(t *testing.T) { item1 := "item1" item2 := "item2" - blacklist, _ := New(map[string]struct{}{}, map[string]struct{}{item1: {}, item2: {}}) - actualStatusString := blacklist.Status() - expectedRegexPattern := `^blacklisting the following items: (item1|item2), (item2|item1)$` + denylist, _ := New(map[string]struct{}{}, map[string]struct{}{item1: {}, item2: {}}) + actualStatusString := denylist.Status() + expectedRegexPattern := `^Excluding the following lists that were on denylist: (item1|item2), (item2|item1)$` matched, _ := regexp.MatchString(expectedRegexPattern, actualStatusString) if !matched { t.Errorf("expected status %q but got %q", expectedRegexPattern, actualStatusString) diff --git a/pkg/app/server.go b/pkg/app/server.go new file mode 100644 index 0000000000..2af7e6d446 --- /dev/null +++ b/pkg/app/server.go @@ -0,0 +1,451 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "context" + "crypto/md5" //nolint:gosec + "encoding/binary" + "fmt" + "net" + "net/http" + "net/http/pprof" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "time" + + "gopkg.in/yaml.v3" + + "github.com/oklog/run" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/common/version" + "github.com/prometheus/exporter-toolkit/web" + vpaclientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned" + clientset "k8s.io/client-go/kubernetes" + _ "k8s.io/client-go/plugin/pkg/client/auth" // Initialize common client auth plugins. + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + + "k8s.io/kube-state-metrics/v2/internal/store" + "k8s.io/kube-state-metrics/v2/pkg/allowdenylist" + "k8s.io/kube-state-metrics/v2/pkg/customresource" + "k8s.io/kube-state-metrics/v2/pkg/customresourcestate" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" + "k8s.io/kube-state-metrics/v2/pkg/metricshandler" + "k8s.io/kube-state-metrics/v2/pkg/optin" + "k8s.io/kube-state-metrics/v2/pkg/options" + "k8s.io/kube-state-metrics/v2/pkg/util/proc" +) + +const ( + metricsPath = "/metrics" + healthzPath = "/healthz" +) + +// promLogger implements promhttp.Logger +type promLogger struct{} + +func (pl promLogger) Println(v ...interface{}) { + klog.Error(v...) +} + +// promLogger implements the Logger interface +func (pl promLogger) Log(v ...interface{}) error { + klog.Info(v...) + return nil +} + +// RunKubeStateMetricsWrapper runs KSM with context cancellation. +func RunKubeStateMetricsWrapper(ctx context.Context, opts *options.Options) error { + err := RunKubeStateMetrics(ctx, opts) + if ctx.Err() == context.Canceled { + klog.Infoln("Restarting: kube-state-metrics, metrics will be reset") + return nil + } + return err +} + +// RunKubeStateMetrics will build and run the kube-state-metrics. +// Any out-of-tree custom resource metrics could be registered by newing a registry factory +// which implements customresource.RegistryFactory and pass all factories into this function. +func RunKubeStateMetrics(ctx context.Context, opts *options.Options) error { + promLogger := promLogger{} + + storeBuilder := store.NewBuilder() + + ksmMetricsRegistry := prometheus.NewRegistry() + ksmMetricsRegistry.MustRegister(version.NewCollector("kube_state_metrics")) + durationVec := promauto.With(ksmMetricsRegistry).NewHistogramVec( + prometheus.HistogramOpts{ + Name: "http_request_duration_seconds", + Help: "A histogram of requests for kube-state-metrics metrics handler.", + Buckets: prometheus.DefBuckets, + ConstLabels: prometheus.Labels{"handler": "metrics"}, + }, []string{"method"}, + ) + configHash := promauto.With(ksmMetricsRegistry).NewGaugeVec( + prometheus.GaugeOpts{ + Name: "kube_state_metrics_config_hash", + Help: "Hash of the currently loaded configuration.", + }, []string{"type", "filename"}) + configSuccess := promauto.With(ksmMetricsRegistry).NewGaugeVec( + prometheus.GaugeOpts{ + Name: "kube_state_metrics_last_config_reload_successful", + Help: "Whether the last configuration reload attempt was successful.", + }, []string{"type", "filename"}) + configSuccessTime := promauto.With(ksmMetricsRegistry).NewGaugeVec( + prometheus.GaugeOpts{ + Name: "kube_state_metrics_last_config_reload_success_timestamp_seconds", + Help: "Timestamp of the last successful configuration reload.", + }, []string{"type", "filename"}) + + storeBuilder.WithMetrics(ksmMetricsRegistry) + + got := options.GetConfigFile(*opts) + if got != "" { + configFile, err := os.ReadFile(filepath.Clean(got)) + if err != nil { + return fmt.Errorf("failed to read opts config file: %v", err) + } + // NOTE: Config value will override default values of intersecting options. + err = yaml.Unmarshal(configFile, opts) + if err != nil { + // DO NOT end the process. + // We want to allow the user to still be able to fix the misconfigured config (redeploy or edit the configmaps) and reload KSM automatically once that's done. + klog.ErrorS(err, "failed to unmarshal opts config file") + // Wait for the next reload. + klog.InfoS("misconfigured config detected, KSM will automatically reload on next write to the config") + klog.InfoS("waiting for config to be fixed") + configSuccess.WithLabelValues("config", filepath.Clean(got)).Set(0) + <-ctx.Done() + } else { + configSuccess.WithLabelValues("config", filepath.Clean(got)).Set(1) + configSuccessTime.WithLabelValues("config", filepath.Clean(got)).SetToCurrentTime() + hash := md5HashAsMetricValue(configFile) + configHash.WithLabelValues("config", filepath.Clean(got)).Set(hash) + } + } + + // Loading custom resource state configuration from cli argument or config file + config, err := resolveCustomResourceConfig(opts) + if err != nil { + return err + } + + var factories []customresource.RegistryFactory + + if config != nil { + factories, err = customresourcestate.FromConfig(config) + if err != nil { + return fmt.Errorf("Parsing from Custom Resource State Metrics file failed: %v", err) + } + } + storeBuilder.WithCustomResourceStoreFactories(factories...) + + if opts.CustomResourceConfigFile != "" { + crcFile, err := os.ReadFile(filepath.Clean(opts.CustomResourceConfigFile)) + if err != nil { + return fmt.Errorf("failed to read custom resource config file: %v", err) + } + configSuccess.WithLabelValues("customresourceconfig", filepath.Clean(opts.CustomResourceConfigFile)).Set(1) + configSuccessTime.WithLabelValues("customresourceconfig", filepath.Clean(opts.CustomResourceConfigFile)).SetToCurrentTime() + hash := md5HashAsMetricValue(crcFile) + configHash.WithLabelValues("customresourceconfig", filepath.Clean(opts.CustomResourceConfigFile)).Set(hash) + + } + + resources := make([]string, len(factories)) + + for i, factory := range factories { + resources[i] = factory.Name() + } + + switch { + case len(opts.Resources) == 0 && !opts.CustomResourcesOnly: + resources = append(resources, options.DefaultResources.AsSlice()...) + klog.InfoS("Used default resources") + case opts.CustomResourcesOnly: + // enable custom resource only + klog.InfoS("Used CRD resources only", "resources", resources) + default: + resources = append(resources, opts.Resources.AsSlice()...) + klog.InfoS("Used resources", "resources", resources) + } + + if err := storeBuilder.WithEnabledResources(resources); err != nil { + return fmt.Errorf("failed to set up resources: %v", err) + } + + namespaces := opts.Namespaces.GetNamespaces() + nsFieldSelector := namespaces.GetExcludeNSFieldSelector(opts.NamespacesDenylist) + nodeFieldSelector := opts.Node.GetNodeFieldSelector() + merged, err := storeBuilder.MergeFieldSelectors([]string{nsFieldSelector, nodeFieldSelector}) + if err != nil { + return err + } + storeBuilder.WithNamespaces(namespaces) + storeBuilder.WithFieldSelectorFilter(merged) + + allowDenyList, err := allowdenylist.New(opts.MetricAllowlist, opts.MetricDenylist) + if err != nil { + return err + } + + err = allowDenyList.Parse() + if err != nil { + return fmt.Errorf("error initializing the allowdeny list: %v", err) + } + + klog.InfoS("Metric allow-denylisting", "allowDenyStatus", allowDenyList.Status()) + + optInMetricFamilyFilter, err := optin.NewMetricFamilyFilter(opts.MetricOptInList) + if err != nil { + return fmt.Errorf("error initializing the opt-in metric list: %v", err) + } + + if optInMetricFamilyFilter.Count() > 0 { + klog.InfoS("Metrics which were opted into", "optInMetricsFamilyStatus", optInMetricFamilyFilter.Status()) + } + + storeBuilder.WithFamilyGeneratorFilter(generator.NewCompositeFamilyGeneratorFilter( + allowDenyList, + optInMetricFamilyFilter, + )) + + storeBuilder.WithUsingAPIServerCache(opts.UseAPIServerCache) + storeBuilder.WithGenerateStoresFunc(storeBuilder.DefaultGenerateStoresFunc()) + storeBuilder.WithGenerateCustomResourceStoresFunc(storeBuilder.DefaultGenerateCustomResourceStoresFunc()) + + proc.StartReaper() + + kubeClient, vpaClient, customResourceClients, err := createKubeClient(opts.Apiserver, opts.Kubeconfig, factories...) + if err != nil { + return fmt.Errorf("failed to create client: %v", err) + } + storeBuilder.WithKubeClient(kubeClient) + storeBuilder.WithVPAClient(vpaClient) + storeBuilder.WithCustomResourceClients(customResourceClients) + storeBuilder.WithSharding(opts.Shard, opts.TotalShards) + storeBuilder.WithAllowAnnotations(opts.AnnotationsAllowList) + if err := storeBuilder.WithAllowLabels(opts.LabelsAllowList); err != nil { + return fmt.Errorf("failed to set up labels allowlist: %v", err) + } + + ksmMetricsRegistry.MustRegister( + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + collectors.NewGoCollector(), + ) + + var g run.Group + + m := metricshandler.New( + opts, + kubeClient, + storeBuilder, + opts.EnableGZIPEncoding, + ) + // Run MetricsHandler + { + ctxMetricsHandler, cancel := context.WithCancel(ctx) + g.Add(func() error { + return m.Run(ctxMetricsHandler) + }, func(error) { + cancel() + }) + } + + tlsConfig := opts.TLSConfig + + telemetryMux := buildTelemetryServer(ksmMetricsRegistry) + telemetryListenAddress := net.JoinHostPort(opts.TelemetryHost, strconv.Itoa(opts.TelemetryPort)) + telemetryServer := http.Server{ + Handler: telemetryMux, + ReadHeaderTimeout: 5 * time.Second} + telemetryFlags := web.FlagConfig{ + WebListenAddresses: &[]string{telemetryListenAddress}, + WebSystemdSocket: new(bool), + WebConfigFile: &tlsConfig, + } + + metricsMux := buildMetricsServer(m, durationVec) + metricsServerListenAddress := net.JoinHostPort(opts.Host, strconv.Itoa(opts.Port)) + metricsServer := http.Server{ + Handler: metricsMux, + ReadHeaderTimeout: 5 * time.Second} + + metricsFlags := web.FlagConfig{ + WebListenAddresses: &[]string{metricsServerListenAddress}, + WebSystemdSocket: new(bool), + WebConfigFile: &tlsConfig, + } + + // Run Telemetry server + { + g.Add(func() error { + klog.InfoS("Started kube-state-metrics self metrics server", "telemetryAddress", telemetryListenAddress) + return web.ListenAndServe(&telemetryServer, &telemetryFlags, promLogger) + }, func(error) { + ctxShutDown, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + telemetryServer.Shutdown(ctxShutDown) + }) + } + // Run Metrics server + { + g.Add(func() error { + klog.InfoS("Started metrics server", "metricsServerAddress", metricsServerListenAddress) + return web.ListenAndServe(&metricsServer, &metricsFlags, promLogger) + }, func(error) { + ctxShutDown, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + metricsServer.Shutdown(ctxShutDown) + }) + } + + if err := g.Run(); err != nil { + return fmt.Errorf("run server group error: %v", err) + } + klog.InfoS("Exited") + return nil +} + +func createKubeClient(apiserver string, kubeconfig string, factories ...customresource.RegistryFactory) (clientset.Interface, vpaclientset.Interface, map[string]interface{}, error) { + config, err := clientcmd.BuildConfigFromFlags(apiserver, kubeconfig) + if err != nil { + return nil, nil, nil, err + } + + config.UserAgent = fmt.Sprintf("%s/%s (%s/%s) kubernetes/%s", "kube-state-metrics", version.Version, runtime.GOOS, runtime.GOARCH, version.Revision) + config.AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json" + config.ContentType = "application/vnd.kubernetes.protobuf" + + kubeClient, err := clientset.NewForConfig(config) + if err != nil { + return nil, nil, nil, err + } + + vpaClient, err := vpaclientset.NewForConfig(config) + if err != nil { + return nil, nil, nil, err + } + + customResourceClients := make(map[string]interface{}, len(factories)) + for _, f := range factories { + customResourceClient, err := f.CreateClient(config) + if err != nil { + return nil, nil, nil, err + } + customResourceClients[f.Name()] = customResourceClient + } + + // Informers don't seem to do a good job logging error messages when it + // can't reach the server, making debugging hard. This makes it easier to + // figure out if apiserver is configured incorrectly. + klog.InfoS("Tested communication with server") + v, err := kubeClient.Discovery().ServerVersion() + if err != nil { + return nil, nil, nil, fmt.Errorf("error while trying to communicate with apiserver: %w", err) + } + klog.InfoS("Run with Kubernetes cluster version", "major", v.Major, "minor", v.Minor, "gitVersion", v.GitVersion, "gitTreeState", v.GitTreeState, "gitCommit", v.GitCommit, "platform", v.Platform) + klog.InfoS("Communication with server successful") + + return kubeClient, vpaClient, customResourceClients, nil +} + +func buildTelemetryServer(registry prometheus.Gatherer) *http.ServeMux { + mux := http.NewServeMux() + + // Add metricsPath + mux.Handle(metricsPath, promhttp.HandlerFor(registry, promhttp.HandlerOpts{ErrorLog: promLogger{}})) + // Add index + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(` + Kube-State-Metrics Metrics Server + +

Kube-State-Metrics Metrics

+ + + `)) + }) + return mux +} + +func buildMetricsServer(m *metricshandler.MetricsHandler, durationObserver prometheus.ObserverVec) *http.ServeMux { + mux := http.NewServeMux() + + // TODO: This doesn't belong into serveMetrics + mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index)) + mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline)) + mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile)) + mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol)) + mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace)) + + mux.Handle(metricsPath, promhttp.InstrumentHandlerDuration(durationObserver, m)) + + // Add healthzPath + mux.HandleFunc(healthzPath, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(http.StatusText(http.StatusOK))) + }) + // Add index + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(` + Kube Metrics Server + +

Kube Metrics

+ + + `)) + }) + return mux +} + +// md5HashAsMetricValue creates an md5 hash and returns the most significant bytes that fit into a float64 +// Taken from https://github.com/prometheus/alertmanager/blob/6ef6e6868dbeb7984d2d577dd4bf75c65bf1904f/config/coordinator.go#L149 +func md5HashAsMetricValue(data []byte) float64 { + sum := md5.Sum(data) //nolint:gosec + // We only want 48 bits as a float64 only has a 53 bit mantissa. + smallSum := sum[0:6] + bytes := make([]byte, 8) + copy(bytes, smallSum) + return float64(binary.LittleEndian.Uint64(bytes)) +} + +func resolveCustomResourceConfig(opts *options.Options) (customresourcestate.ConfigDecoder, error) { + if s := opts.CustomResourceConfig; s != "" { + return yaml.NewDecoder(strings.NewReader(s)), nil + } + if file := opts.CustomResourceConfigFile; file != "" { + f, err := os.Open(filepath.Clean(file)) + if err != nil { + return nil, fmt.Errorf("Custom Resource State Metrics file could not be opened: %v", err) + } + return yaml.NewDecoder(f), nil + } + return nil, nil +} diff --git a/pkg/app/server_test.go b/pkg/app/server_test.go new file mode 100644 index 0000000000..61fa598057 --- /dev/null +++ b/pkg/app/server_test.go @@ -0,0 +1,962 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http/httptest" + "sort" + "strconv" + "strings" + "testing" + "time" + + "k8s.io/kube-state-metrics/v2/pkg/optin" + + "github.com/prometheus/client_golang/prometheus" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + samplev1alpha1 "k8s.io/sample-controller/pkg/apis/samplecontroller/v1alpha1" + samplefake "k8s.io/sample-controller/pkg/generated/clientset/versioned/fake" + + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/internal/store" + "k8s.io/kube-state-metrics/v2/pkg/allowdenylist" + "k8s.io/kube-state-metrics/v2/pkg/customresource" + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" + "k8s.io/kube-state-metrics/v2/pkg/metricshandler" + "k8s.io/kube-state-metrics/v2/pkg/options" +) + +func BenchmarkKubeStateMetrics(b *testing.B) { + fixtureMultiplier := 1000 + requestCount := 1000 + + b.Logf( + "starting kube-state-metrics benchmark with fixtureMultiplier %v and requestCount %v", + fixtureMultiplier, + requestCount, + ) + + kubeClient := fake.NewSimpleClientset() + + if err := injectFixtures(kubeClient, fixtureMultiplier); err != nil { + b.Errorf("error injecting resources: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + reg := prometheus.NewRegistry() + + builder := store.NewBuilder() + builder.WithMetrics(reg) + err := builder.WithEnabledResources(options.DefaultResources.AsSlice()) + if err != nil { + b.Fatal(err) + } + builder.WithKubeClient(kubeClient) + builder.WithSharding(0, 1) + builder.WithContext(ctx) + builder.WithNamespaces(options.DefaultNamespaces) + builder.WithGenerateStoresFunc(builder.DefaultGenerateStoresFunc()) + + allowDenyListFilter, err := allowdenylist.New(map[string]struct{}{}, map[string]struct{}{}) + if err != nil { + b.Fatal(err) + } + + builder.WithFamilyGeneratorFilter(generator.NewCompositeFamilyGeneratorFilter( + allowDenyListFilter, + )) + + builder.WithAllowAnnotations(map[string][]string{}) + builder.WithAllowLabels(map[string][]string{}) + + // This test is not suitable to be compared in terms of time, as it includes + // a one second wait. Use for memory allocation comparisons, profiling, ... + handler := metricshandler.New(&options.Options{}, kubeClient, builder, false) + b.Run("GenerateMetrics", func(b *testing.B) { + handler.ConfigureSharding(ctx, 0, 1) + + // Wait for caches to fill + time.Sleep(time.Second) + }) + + req := httptest.NewRequest("GET", "http://localhost:8080/metrics", nil) + + b.Run("MakeRequests", func(b *testing.B) { + var accumulatedContentLength int + + for i := 0; i < requestCount; i++ { + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + resp := w.Result() + if resp.StatusCode != 200 { + b.Fatalf("expected 200 status code but got %v", resp.StatusCode) + } + + b.StopTimer() + buf := bytes.Buffer{} + _, err := buf.ReadFrom(resp.Body) + if err != nil { + b.Fatal(err) + } + accumulatedContentLength += buf.Len() + b.StartTimer() + } + + b.SetBytes(int64(accumulatedContentLength)) + }) +} + +// TestFullScrapeCycle is a simple smoke test covering the entire cycle from +// cache filling to scraping. +func TestFullScrapeCycle(t *testing.T) { + t.Parallel() + + kubeClient := fake.NewSimpleClientset() + + err := pod(kubeClient, 0) + if err != nil { + t.Fatalf("failed to insert sample pod %v", err.Error()) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + reg := prometheus.NewRegistry() + builder := store.NewBuilder() + builder.WithMetrics(reg) + err = builder.WithEnabledResources(options.DefaultResources.AsSlice()) + if err != nil { + t.Fatal(err) + } + builder.WithKubeClient(kubeClient) + builder.WithNamespaces(options.DefaultNamespaces) + builder.WithGenerateStoresFunc(builder.DefaultGenerateStoresFunc()) + + l, err := allowdenylist.New(map[string]struct{}{}, map[string]struct{}{}) + if err != nil { + t.Fatal(err) + } + + optInMetrics := make(map[string]struct{}) + optInMetricFamilyFilter, err := optin.NewMetricFamilyFilter(optInMetrics) + if err != nil { + t.Fatal(err) + } + + builder.WithFamilyGeneratorFilter(generator.NewCompositeFamilyGeneratorFilter( + l, + optInMetricFamilyFilter, + )) + builder.WithAllowLabels(map[string][]string{ + "kube_pod_labels": { + "namespace", + "pod", + "uid", + }, + }) + + handler := metricshandler.New(&options.Options{}, kubeClient, builder, false) + handler.ConfigureSharding(ctx, 0, 1) + + // Wait for caches to fill + time.Sleep(time.Second) + + req := httptest.NewRequest("GET", "http://localhost:8080/metrics", nil) + + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + resp := w.Result() + if resp.StatusCode != 200 { + t.Fatalf("expected 200 status code but got %v", resp.StatusCode) + } + + body, _ := io.ReadAll(resp.Body) + + expected := `# HELP kube_pod_annotations Kubernetes annotations converted to Prometheus labels. +# HELP kube_pod_completion_time [STABLE] Completion time in unix timestamp for a pod. +# HELP kube_pod_container_info [STABLE] Information about a container in a pod. +# HELP kube_pod_container_resource_limits The number of requested limit resource by a container. It is recommended to use the kube_pod_resource_limits metric exposed by kube-scheduler instead, as it is more precise. +# HELP kube_pod_container_resource_requests The number of requested request resource by a container. It is recommended to use the kube_pod_resource_requests metric exposed by kube-scheduler instead, as it is more precise. +# HELP kube_pod_container_state_started [STABLE] Start time in unix timestamp for a pod container. +# HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state. +# HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state. +# HELP kube_pod_container_status_ready [STABLE] Describes whether the containers readiness check succeeded. +# HELP kube_pod_container_status_restarts_total [STABLE] The number of container restarts per container. +# HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state. +# HELP kube_pod_container_status_terminated [STABLE] Describes whether the container is currently in terminated state. +# HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state. +# HELP kube_pod_container_status_waiting [STABLE] Describes whether the container is currently in waiting state. +# HELP kube_pod_container_status_waiting_reason [STABLE] Describes the reason the container is currently in waiting state. +# HELP kube_pod_created [STABLE] Unix creation timestamp +# HELP kube_pod_deletion_timestamp Unix deletion timestamp +# HELP kube_pod_info [STABLE] Information about pod. +# HELP kube_pod_init_container_info [STABLE] Information about an init container in a pod. +# HELP kube_pod_init_container_resource_limits The number of requested limit resource by an init container. +# HELP kube_pod_init_container_resource_requests The number of requested request resource by an init container. +# HELP kube_pod_init_container_status_last_terminated_reason Describes the last reason the init container was in terminated state. +# HELP kube_pod_init_container_status_ready [STABLE] Describes whether the init containers readiness check succeeded. +# HELP kube_pod_init_container_status_restarts_total [STABLE] The number of restarts for the init container. +# HELP kube_pod_init_container_status_running [STABLE] Describes whether the init container is currently in running state. +# HELP kube_pod_init_container_status_terminated [STABLE] Describes whether the init container is currently in terminated state. +# HELP kube_pod_init_container_status_terminated_reason Describes the reason the init container is currently in terminated state. +# HELP kube_pod_init_container_status_waiting [STABLE] Describes whether the init container is currently in waiting state. +# HELP kube_pod_init_container_status_waiting_reason Describes the reason the init container is currently in waiting state. +# HELP kube_pod_ips Pod IP addresses +# HELP kube_pod_labels [STABLE] Kubernetes labels converted to Prometheus labels. +# HELP kube_pod_overhead_cpu_cores The pod overhead in regards to cpu cores associated with running a pod. +# HELP kube_pod_overhead_memory_bytes The pod overhead in regards to memory associated with running a pod. +# HELP kube_pod_runtimeclass_name_info The runtimeclass associated with the pod. +# HELP kube_pod_owner [STABLE] Information about the Pod's owner. +# HELP kube_pod_restart_policy [STABLE] Describes the restart policy in use by this pod. +# HELP kube_pod_spec_volumes_persistentvolumeclaims_info [STABLE] Information about persistentvolumeclaim volumes in a pod. +# HELP kube_pod_spec_volumes_persistentvolumeclaims_readonly [STABLE] Describes whether a persistentvolumeclaim is mounted read only. +# HELP kube_pod_start_time [STABLE] Start time in unix timestamp for a pod. +# HELP kube_pod_status_container_ready_time Readiness achieved time in unix timestamp for a pod containers. +# HELP kube_pod_status_qos_class The pods current qosClass. +# HELP kube_pod_status_phase [STABLE] The pods current phase. +# HELP kube_pod_status_ready_time Readiness achieved time in unix timestamp for a pod. +# HELP kube_pod_status_ready [STABLE] Describes whether the pod is ready to serve requests. +# HELP kube_pod_status_reason The pod status reasons +# HELP kube_pod_status_scheduled [STABLE] Describes the status of the scheduling process for the pod. +# HELP kube_pod_status_scheduled_time [STABLE] Unix timestamp when pod moved into scheduled status +# HELP kube_pod_status_unschedulable [STABLE] Describes the unschedulable status for the pod. +# HELP kube_pod_tolerations Information about the pod tolerations +# TYPE kube_pod_annotations gauge +# TYPE kube_pod_completion_time gauge +# TYPE kube_pod_container_info gauge +# TYPE kube_pod_container_resource_limits gauge +# TYPE kube_pod_container_resource_requests gauge +# TYPE kube_pod_container_state_started gauge +# TYPE kube_pod_container_status_last_terminated_exitcode gauge +# TYPE kube_pod_container_status_last_terminated_reason gauge +# TYPE kube_pod_container_status_ready gauge +# TYPE kube_pod_container_status_restarts_total counter +# TYPE kube_pod_container_status_running gauge +# TYPE kube_pod_container_status_terminated gauge +# TYPE kube_pod_container_status_terminated_reason gauge +# TYPE kube_pod_container_status_waiting gauge +# TYPE kube_pod_container_status_waiting_reason gauge +# TYPE kube_pod_created gauge +# TYPE kube_pod_deletion_timestamp gauge +# TYPE kube_pod_info gauge +# TYPE kube_pod_init_container_info gauge +# TYPE kube_pod_init_container_resource_limits gauge +# TYPE kube_pod_init_container_resource_requests gauge +# TYPE kube_pod_init_container_status_last_terminated_reason gauge +# TYPE kube_pod_init_container_status_ready gauge +# TYPE kube_pod_init_container_status_restarts_total counter +# TYPE kube_pod_init_container_status_running gauge +# TYPE kube_pod_init_container_status_terminated gauge +# TYPE kube_pod_init_container_status_terminated_reason gauge +# TYPE kube_pod_init_container_status_waiting gauge +# TYPE kube_pod_init_container_status_waiting_reason gauge +# TYPE kube_pod_ips gauge +# TYPE kube_pod_labels gauge +# TYPE kube_pod_overhead_cpu_cores gauge +# TYPE kube_pod_overhead_memory_bytes gauge +# TYPE kube_pod_runtimeclass_name_info gauge +# TYPE kube_pod_owner gauge +# TYPE kube_pod_restart_policy gauge +# TYPE kube_pod_spec_volumes_persistentvolumeclaims_info gauge +# TYPE kube_pod_spec_volumes_persistentvolumeclaims_readonly gauge +# TYPE kube_pod_start_time gauge +# TYPE kube_pod_status_container_ready_time gauge +# TYPE kube_pod_status_phase gauge +# TYPE kube_pod_status_qos_class gauge +# TYPE kube_pod_status_ready gauge +# TYPE kube_pod_status_ready_time gauge +# TYPE kube_pod_status_reason gauge +# TYPE kube_pod_status_scheduled gauge +# TYPE kube_pod_status_scheduled_time gauge +# TYPE kube_pod_status_unschedulable gauge +# TYPE kube_pod_tolerations gauge +kube_pod_annotations{namespace="default",pod="pod0",uid="abc-0"} 1 +kube_pod_container_info{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",image_spec="k8s.gcr.io/hyperkube2_spec",image="k8s.gcr.io/hyperkube2",image_id="docker://sha256:bbb",container_id="docker://cd456"} 1 +kube_pod_container_info{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2",image_spec="k8s.gcr.io/hyperkube3_spec",image="k8s.gcr.io/hyperkube3",image_id="docker://sha256:ccc",container_id="docker://ef789"} 1 +kube_pod_container_resource_limits{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",node="node1",resource="cpu",unit="core"} 0.2 +kube_pod_container_resource_limits{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",node="node1",resource="ephemeral_storage",unit="byte"} 3e+08 +kube_pod_container_resource_limits{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",node="node1",resource="memory",unit="byte"} 1e+08 +kube_pod_container_resource_limits{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",node="node1",resource="nvidia_com_gpu",unit="integer"} 1 +kube_pod_container_resource_limits{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",node="node1",resource="storage",unit="byte"} 4e+08 +kube_pod_container_resource_limits{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2",node="node1",resource="cpu",unit="core"} 0.3 +kube_pod_container_resource_limits{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2",node="node1",resource="memory",unit="byte"} 2e+08 +kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",node="node1",resource="cpu",unit="core"} 0.2 +kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",node="node1",resource="ephemeral_storage",unit="byte"} 3e+08 +kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",node="node1",resource="memory",unit="byte"} 1e+08 +kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",node="node1",resource="nvidia_com_gpu",unit="integer"} 1 +kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",node="node1",resource="storage",unit="byte"} 4e+08 +kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2",node="node1",resource="cpu",unit="core"} 0.3 +kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2",node="node1",resource="memory",unit="byte"} 2e+08 +kube_pod_container_status_last_terminated_exitcode{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 137 +kube_pod_container_status_last_terminated_reason{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",reason="OOMKilled"} 1 +kube_pod_container_status_ready{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 0 +kube_pod_container_status_ready{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2"} 0 +kube_pod_container_status_restarts_total{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 0 +kube_pod_container_status_restarts_total{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2"} 0 +kube_pod_container_status_running{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 0 +kube_pod_container_status_running{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2"} 0 +kube_pod_container_status_terminated{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 0 +kube_pod_container_status_terminated{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2"} 0 +kube_pod_container_status_waiting_reason{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",reason="CrashLoopBackOff"} 1 +kube_pod_container_status_waiting{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 1 +kube_pod_container_status_waiting{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2"} 0 +kube_pod_created{namespace="default",pod="pod0",uid="abc-0"} 1.5e+09 +kube_pod_info{namespace="default",pod="pod0",uid="abc-0",host_ip="1.1.1.1",pod_ip="1.2.3.4",node="node1",created_by_kind="",created_by_name="",priority_class="",host_network="false"} 1 +kube_pod_labels{namespace="default",pod="pod0",uid="abc-0"} 1 +kube_pod_owner{namespace="default",pod="pod0",uid="abc-0",owner_kind="",owner_name="",owner_is_controller=""} 1 +kube_pod_restart_policy{namespace="default",pod="pod0",uid="abc-0",type="Always"} 1 +kube_pod_status_phase{namespace="default",pod="pod0",uid="abc-0",phase="Failed"} 0 +kube_pod_status_phase{namespace="default",pod="pod0",uid="abc-0",phase="Pending"} 0 +kube_pod_status_phase{namespace="default",pod="pod0",uid="abc-0",phase="Running"} 1 +kube_pod_status_phase{namespace="default",pod="pod0",uid="abc-0",phase="Succeeded"} 0 +kube_pod_status_phase{namespace="default",pod="pod0",uid="abc-0",phase="Unknown"} 0 +kube_pod_status_reason{namespace="default",pod="pod0",uid="abc-0",reason="Evicted"} 0 +kube_pod_status_reason{namespace="default",pod="pod0",uid="abc-0",reason="NodeAffinity"} 0 +kube_pod_status_reason{namespace="default",pod="pod0",uid="abc-0",reason="NodeLost"} 0 +kube_pod_status_reason{namespace="default",pod="pod0",uid="abc-0",reason="Shutdown"} 0 +kube_pod_status_reason{namespace="default",pod="pod0",uid="abc-0",reason="UnexpectedAdmissionError"} 0 +` + + expectedSplit := strings.Split(strings.TrimSpace(expected), "\n") + sort.Strings(expectedSplit) + + gotSplit := strings.Split(strings.TrimSpace(string(body)), "\n") + + gotFiltered := []string{} + for _, l := range gotSplit { + if strings.Contains(l, "kube_pod_") { + gotFiltered = append(gotFiltered, l) + } + } + + sort.Strings(gotFiltered) + + if len(expectedSplit) != len(gotFiltered) { + fmt.Println(len(expectedSplit)) + fmt.Println(len(gotFiltered)) + t.Fatalf("expected different output length, expected \n\n%s\n\ngot\n\n%s", expected, strings.Join(gotFiltered, "\n")) + } + + for i := 0; i < len(expectedSplit); i++ { + if expectedSplit[i] != gotFiltered[i] { + t.Fatalf("expected:\n\n%v\n, but got:\n\n%v", expectedSplit[i], gotFiltered[i]) + } + } + + telemetryMux := buildTelemetryServer(reg) + + req2 := httptest.NewRequest("GET", "http://localhost:8081/metrics", nil) + + w2 := httptest.NewRecorder() + telemetryMux.ServeHTTP(w2, req2) + + resp2 := w2.Result() + if resp.StatusCode != 200 { + t.Fatalf("expected 200 status code but got %v", resp.StatusCode) + } + + body2, _ := io.ReadAll(resp2.Body) + + expected2 := `# HELP kube_state_metrics_shard_ordinal Current sharding ordinal/index of this instance +# HELP kube_state_metrics_total_shards Number of total shards this instance is aware of +# TYPE kube_state_metrics_shard_ordinal gauge +# TYPE kube_state_metrics_total_shards gauge +kube_state_metrics_shard_ordinal{shard_ordinal="0"} 0 +kube_state_metrics_total_shards 1 +` + + expectedSplit2 := strings.Split(strings.TrimSpace(expected2), "\n") + sort.Strings(expectedSplit2) + + gotSplit2 := strings.Split(strings.TrimSpace(string(body2)), "\n") + + gotFiltered2 := []string{} + for _, l := range gotSplit2 { + if strings.Contains(l, "_shard") { + gotFiltered2 = append(gotFiltered2, l) + } + } + + sort.Strings(gotFiltered2) + + if len(expectedSplit2) != len(gotFiltered2) { + fmt.Println(len(expectedSplit2)) + fmt.Println(len(gotFiltered2)) + t.Fatalf("expected different output length, expected \n\n%s\n\ngot\n\n%s", expected2, strings.Join(gotFiltered2, "\n")) + } + + for i := 0; i < len(expectedSplit2); i++ { + if expectedSplit2[i] != gotFiltered2[i] { + t.Fatalf("expected:\n\n%v\n, but got:\n\n%v", expectedSplit2[i], gotFiltered2[i]) + } + } +} + +// TestShardingEquivalenceScrapeCycle is a simple smoke test covering the entire cycle from +// cache filling to scraping comparing a sharded with an unsharded setup. +func TestShardingEquivalenceScrapeCycle(t *testing.T) { + t.Parallel() + + kubeClient := fake.NewSimpleClientset() + + for i := 0; i < 10; i++ { + err := pod(kubeClient, i) + if err != nil { + t.Fatalf("failed to insert sample pod %v", err.Error()) + } + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + l, err := allowdenylist.New(map[string]struct{}{}, map[string]struct{}{}) + if err != nil { + t.Fatal(err) + } + + reg := prometheus.NewRegistry() + unshardedBuilder := store.NewBuilder() + unshardedBuilder.WithMetrics(reg) + err = unshardedBuilder.WithEnabledResources(options.DefaultResources.AsSlice()) + if err != nil { + t.Fatal(err) + } + unshardedBuilder.WithKubeClient(kubeClient) + unshardedBuilder.WithNamespaces(options.DefaultNamespaces) + unshardedBuilder.WithFamilyGeneratorFilter(l) + unshardedBuilder.WithAllowLabels(map[string][]string{}) + unshardedBuilder.WithGenerateStoresFunc(unshardedBuilder.DefaultGenerateStoresFunc()) + + unshardedHandler := metricshandler.New(&options.Options{}, kubeClient, unshardedBuilder, false) + unshardedHandler.ConfigureSharding(ctx, 0, 1) + + regShard1 := prometheus.NewRegistry() + shardedBuilder1 := store.NewBuilder() + shardedBuilder1.WithMetrics(regShard1) + err = shardedBuilder1.WithEnabledResources(options.DefaultResources.AsSlice()) + if err != nil { + t.Fatal(err) + } + shardedBuilder1.WithKubeClient(kubeClient) + shardedBuilder1.WithNamespaces(options.DefaultNamespaces) + shardedBuilder1.WithFamilyGeneratorFilter(l) + shardedBuilder1.WithAllowLabels(map[string][]string{}) + shardedBuilder1.WithGenerateStoresFunc(shardedBuilder1.DefaultGenerateStoresFunc()) + + shardedHandler1 := metricshandler.New(&options.Options{}, kubeClient, shardedBuilder1, false) + shardedHandler1.ConfigureSharding(ctx, 0, 2) + + regShard2 := prometheus.NewRegistry() + shardedBuilder2 := store.NewBuilder() + shardedBuilder2.WithMetrics(regShard2) + err = shardedBuilder2.WithEnabledResources(options.DefaultResources.AsSlice()) + if err != nil { + t.Fatal(err) + } + shardedBuilder2.WithKubeClient(kubeClient) + shardedBuilder2.WithNamespaces(options.DefaultNamespaces) + shardedBuilder2.WithFamilyGeneratorFilter(l) + shardedBuilder2.WithAllowLabels(map[string][]string{}) + shardedBuilder2.WithGenerateStoresFunc(shardedBuilder2.DefaultGenerateStoresFunc()) + + shardedHandler2 := metricshandler.New(&options.Options{}, kubeClient, shardedBuilder2, false) + shardedHandler2.ConfigureSharding(ctx, 1, 2) + + // Wait for caches to fill + time.Sleep(time.Second) + + // unsharded request as the controlled environment + req := httptest.NewRequest("GET", "http://localhost:8080/metrics", nil) + + w := httptest.NewRecorder() + unshardedHandler.ServeHTTP(w, req) + + resp := w.Result() + if resp.StatusCode != 200 { + t.Fatalf("expected 200 status code but got %v", resp.StatusCode) + } + + body, _ := io.ReadAll(resp.Body) + expected := string(body) + + // sharded requests + // + // request first shard + req = httptest.NewRequest("GET", "http://localhost:8080/metrics", nil) + + w = httptest.NewRecorder() + shardedHandler1.ServeHTTP(w, req) + + resp = w.Result() + if resp.StatusCode != 200 { + t.Fatalf("expected 200 status code but got %v", resp.StatusCode) + } + + body, _ = io.ReadAll(resp.Body) + got1 := string(body) + + // request second shard + req = httptest.NewRequest("GET", "http://localhost:8080/metrics", nil) + + w = httptest.NewRecorder() + shardedHandler2.ServeHTTP(w, req) + + resp = w.Result() + if resp.StatusCode != 200 { + t.Fatalf("expected 200 status code but got %v", resp.StatusCode) + } + + body, _ = io.ReadAll(resp.Body) + got2 := string(body) + + // normalize results: + + expectedSplit := strings.Split(strings.TrimSpace(expected), "\n") + sort.Strings(expectedSplit) + + expectedFiltered := []string{} + for _, l := range expectedSplit { + if strings.HasPrefix(l, "kube_pod_") { + expectedFiltered = append(expectedFiltered, l) + } + } + + got1Split := strings.Split(strings.TrimSpace(got1), "\n") + sort.Strings(got1Split) + + got1Filtered := []string{} + for _, l := range got1Split { + if strings.HasPrefix(l, "kube_pod_") { + got1Filtered = append(got1Filtered, l) + } + } + + got2Split := strings.Split(strings.TrimSpace(got2), "\n") + sort.Strings(got2Split) + + got2Filtered := []string{} + for _, l := range got2Split { + if strings.HasPrefix(l, "kube_pod_") { + got2Filtered = append(got2Filtered, l) + } + } + + // total metrics should be equal + if len(expectedFiltered) != (len(got1Filtered) + len(got2Filtered)) { + t.Fatalf("expected different output length, expected total %d got 1) %d 2) %d", len(expectedFiltered), len(got1Filtered), len(got2Filtered)) + } + // smoke test to test that each shard actually represents a subset + if len(got1Filtered) == 0 { + t.Fatal("shard 1 has 0 metrics when it shouldn't") + } + if len(got2Filtered) == 0 { + t.Fatal("shard 2 has 0 metrics when it shouldn't") + } + + got1Filtered = append(got1Filtered, got2Filtered...) + sort.Strings(got1Filtered) + + for i := 0; i < len(expectedFiltered); i++ { + expected := strings.TrimSpace(expectedFiltered[i]) + got := strings.TrimSpace(got1Filtered[i]) + if expected != got { + t.Fatalf("\n\nexpected:\n\n%q\n\nbut got:\n\n%q\n\n", expected, got) + } + } +} + +// TestCustomResourceExtension is a simple smoke test covering the custom resource metrics collection. +// We use custom resource object samplev1alpha1.Foo in kubernetes/sample-controller as an example. +func TestCustomResourceExtension(t *testing.T) { + kubeClient := fake.NewSimpleClientset() + factories := []customresource.RegistryFactory{new(fooFactory)} + resources := options.DefaultResources.AsSlice() + customResourceClients := make(map[string]interface{}, len(factories)) + // enable custom resource + for _, f := range factories { + resources = append(resources, f.Name()) + customResourceClient, err := f.CreateClient(nil) + if err != nil { + t.Fatalf("Failed to create customResourceClient for foo: %v", err) + } + customResourceClients[f.Name()] = customResourceClient + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + reg := prometheus.NewRegistry() + builder := store.NewBuilder() + builder.WithCustomResourceStoreFactories(factories...) + builder.WithMetrics(reg) + err := builder.WithEnabledResources(resources) + if err != nil { + t.Fatal(err) + } + + builder.WithKubeClient(kubeClient) + builder.WithCustomResourceClients(customResourceClients) + builder.WithNamespaces(options.DefaultNamespaces) + builder.WithGenerateStoresFunc(builder.DefaultGenerateStoresFunc()) + builder.WithGenerateCustomResourceStoresFunc(builder.DefaultGenerateCustomResourceStoresFunc()) + + l, err := allowdenylist.New(map[string]struct{}{}, map[string]struct{}{}) + if err != nil { + t.Fatal(err) + } + builder.WithFamilyGeneratorFilter(l) + builder.WithAllowLabels(map[string][]string{ + "kube_foo_labels": { + "namespace", + "foo", + "uid", + }, + }) + + handler := metricshandler.New(&options.Options{}, kubeClient, builder, false) + handler.ConfigureSharding(ctx, 0, 1) + + // Wait for caches to fill + time.Sleep(time.Second) + + req := httptest.NewRequest("GET", "http://localhost:8080/metrics", nil) + + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + resp := w.Result() + if resp.StatusCode != 200 { + t.Fatalf("expected 200 status code but got %v", resp.StatusCode) + } + + body, _ := io.ReadAll(resp.Body) + + expected := `# HELP kube_foo_spec_replicas Number of desired replicas for a foo. +# HELP kube_foo_status_replicas_available The number of available replicas per foo. +# TYPE kube_foo_spec_replicas gauge +# TYPE kube_foo_status_replicas_available gauge +kube_foo_spec_replicas{namespace="default",foo="foo0"} 0 +kube_foo_spec_replicas{namespace="default",foo="foo1"} 1 +kube_foo_spec_replicas{namespace="default",foo="foo2"} 2 +kube_foo_spec_replicas{namespace="default",foo="foo3"} 3 +kube_foo_spec_replicas{namespace="default",foo="foo4"} 4 +kube_foo_spec_replicas{namespace="default",foo="foo5"} 5 +kube_foo_spec_replicas{namespace="default",foo="foo6"} 6 +kube_foo_spec_replicas{namespace="default",foo="foo7"} 7 +kube_foo_spec_replicas{namespace="default",foo="foo8"} 8 +kube_foo_spec_replicas{namespace="default",foo="foo9"} 9 +kube_foo_status_replicas_available{namespace="default",foo="foo0"} 0 +kube_foo_status_replicas_available{namespace="default",foo="foo1"} 1 +kube_foo_status_replicas_available{namespace="default",foo="foo2"} 2 +kube_foo_status_replicas_available{namespace="default",foo="foo3"} 3 +kube_foo_status_replicas_available{namespace="default",foo="foo5"} 5 +kube_foo_status_replicas_available{namespace="default",foo="foo6"} 6 +kube_foo_status_replicas_available{namespace="default",foo="foo7"} 7 +kube_foo_status_replicas_available{namespace="default",foo="foo8"} 8 +kube_foo_status_replicas_available{namespace="default",foo="foo4"} 4 +kube_foo_status_replicas_available{namespace="default",foo="foo9"} 9 +` + + expectedSplit := strings.Split(strings.TrimSpace(expected), "\n") + sort.Strings(expectedSplit) + + gotSplit := strings.Split(strings.TrimSpace(string(body)), "\n") + + gotFiltered := []string{} + for _, l := range gotSplit { + if strings.Contains(l, "kube_foo_") { + gotFiltered = append(gotFiltered, l) + } + } + + sort.Strings(gotFiltered) + + if len(expectedSplit) != len(gotFiltered) { + fmt.Println(len(expectedSplit)) + fmt.Println(len(gotFiltered)) + t.Fatalf("expected different output length, expected \n\n%s\n\ngot\n\n%s", expected, strings.Join(gotFiltered, "\n")) + } + + for i := 0; i < len(expectedSplit); i++ { + if expectedSplit[i] != gotFiltered[i] { + t.Fatalf("expected:\n\n%v\n, but got:\n\n%v", expectedSplit[i], gotFiltered[i]) + } + } +} + +func injectFixtures(client *fake.Clientset, multiplier int) error { + creators := []func(*fake.Clientset, int) error{ + configMap, + service, + pod, + } + + for _, c := range creators { + for i := 0; i < multiplier; i++ { + err := c(client, i) + + if err != nil { + return err + } + } + } + + return nil +} + +func configMap(client *fake.Clientset, index int) error { + i := strconv.Itoa(index) + + configMap := v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "configmap" + i, + ResourceVersion: "123456", + UID: types.UID("abc-" + i), + }, + } + _, err := client.CoreV1().ConfigMaps(metav1.NamespaceDefault).Create(context.TODO(), &configMap, metav1.CreateOptions{}) + return err +} + +func service(client *fake.Clientset, index int) error { + i := strconv.Itoa(index) + + service := v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "service" + i, + ResourceVersion: "123456", + UID: types.UID("abc-" + i), + }, + } + _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(context.TODO(), &service, metav1.CreateOptions{}) + return err +} + +func pod(client *fake.Clientset, index int) error { + i := strconv.Itoa(index) + + pod := v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod" + i, + CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, + Namespace: "default", + UID: types.UID("abc-" + i), + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyAlways, + NodeName: "node1", + Containers: []v1.Container{ + { + Image: "k8s.gcr.io/hyperkube2_spec", + Name: "pod1_con1", + Resources: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("200m"), + v1.ResourceMemory: resource.MustParse("100M"), + v1.ResourceEphemeralStorage: resource.MustParse("300M"), + v1.ResourceStorage: resource.MustParse("400M"), + v1.ResourceName("nvidia.com/gpu"): resource.MustParse("1"), + }, + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("200m"), + v1.ResourceMemory: resource.MustParse("100M"), + v1.ResourceEphemeralStorage: resource.MustParse("300M"), + v1.ResourceStorage: resource.MustParse("400M"), + v1.ResourceName("nvidia.com/gpu"): resource.MustParse("1"), + }, + }, + }, + { + Image: "k8s.gcr.io/hyperkube3_spec", + Name: "pod1_con2", + Resources: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("300m"), + v1.ResourceMemory: resource.MustParse("200M"), + }, + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("300m"), + v1.ResourceMemory: resource.MustParse("200M"), + }, + }, + }, + }, + }, + Status: v1.PodStatus{ + HostIP: "1.1.1.1", + PodIP: "1.2.3.4", + Phase: v1.PodRunning, + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "pod1_con1", + Image: "k8s.gcr.io/hyperkube2", + ImageID: "docker://sha256:bbb", + ContainerID: "docker://cd456", + State: v1.ContainerState{ + Waiting: &v1.ContainerStateWaiting{ + Reason: "CrashLoopBackOff", + }, + }, + LastTerminationState: v1.ContainerState{ + Terminated: &v1.ContainerStateTerminated{ + Reason: "OOMKilled", + ExitCode: 137, + }, + }, + }, + { + Name: "pod1_con2", + Image: "k8s.gcr.io/hyperkube3", + ImageID: "docker://sha256:ccc", + ContainerID: "docker://ef789", + }, + }, + }, + } + + _, err := client.CoreV1().Pods(metav1.NamespaceDefault).Create(context.TODO(), &pod, metav1.CreateOptions{}) + return err +} + +func foo(client *samplefake.Clientset, index int) error { + i := strconv.Itoa(index) + desiredReplicas := int32(index) + + foo := samplev1alpha1.Foo{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo" + i, + CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)}, + UID: types.UID("abc-" + i), + }, + Spec: samplev1alpha1.FooSpec{ + DeploymentName: "foo" + i, + Replicas: &desiredReplicas, + }, + Status: samplev1alpha1.FooStatus{ + AvailableReplicas: desiredReplicas, + }, + } + + _, err := client.SamplecontrollerV1alpha1().Foos(metav1.NamespaceDefault).Create(context.TODO(), &foo, metav1.CreateOptions{}) + return err +} + +var ( + descFooLabelsDefaultLabels = []string{"namespace", "foo"} +) + +type fooFactory struct{} + +func (f *fooFactory) Name() string { + return "foos" +} + +// CreateClient use fake client set to establish 10 foos. +func (f *fooFactory) CreateClient(cfg *rest.Config) (interface{}, error) { + fooClient := samplefake.NewSimpleClientset() + for i := 0; i < 10; i++ { + err := foo(fooClient, i) + if err != nil { + return nil, fmt.Errorf("failed to insert sample pod %v", err) + } + } + return fooClient, nil +} + +func (f *fooFactory) MetricFamilyGenerators(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_foo_spec_replicas", + "Number of desired replicas for a foo.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapFooFunc(func(f *samplev1alpha1.Foo) *metric.Family { + return &metric.Family{ + Metrics: []*metric.Metric{ + { + Value: float64(*f.Spec.Replicas), + }, + }, + } + }), + ), + *generator.NewFamilyGeneratorWithStability( + "kube_foo_status_replicas_available", + "The number of available replicas per foo.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapFooFunc(func(f *samplev1alpha1.Foo) *metric.Family { + return &metric.Family{ + Metrics: []*metric.Metric{ + { + Value: float64(f.Status.AvailableReplicas), + }, + }, + } + }), + ), + } +} + +func wrapFooFunc(f func(*samplev1alpha1.Foo) *metric.Family) func(interface{}) *metric.Family { + return func(obj interface{}) *metric.Family { + foo := obj.(*samplev1alpha1.Foo) + + metricFamily := f(foo) + + for _, m := range metricFamily.Metrics { + m.LabelKeys = append(descFooLabelsDefaultLabels, m.LabelKeys...) + m.LabelValues = append([]string{foo.Namespace, foo.Name}, m.LabelValues...) + } + + return metricFamily + } +} + +func (f *fooFactory) ExpectedType() interface{} { + return &samplev1alpha1.Foo{} +} + +func (f *fooFactory) ListWatch(customResourceClient interface{}, ns string, fieldSelector string) cache.ListerWatcher { + client := customResourceClient.(*samplefake.Clientset) + return &cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + opts.FieldSelector = fieldSelector + return client.SamplecontrollerV1alpha1().Foos(ns).List(context.Background(), opts) + }, + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + opts.FieldSelector = fieldSelector + return client.SamplecontrollerV1alpha1().Foos(ns).Watch(context.Background(), opts) + }, + } +} diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go new file mode 100644 index 0000000000..828e956f14 --- /dev/null +++ b/pkg/builder/builder.go @@ -0,0 +1,154 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + + "github.com/prometheus/client_golang/prometheus" + vpaclientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + internalstore "k8s.io/kube-state-metrics/v2/internal/store" + ksmtypes "k8s.io/kube-state-metrics/v2/pkg/builder/types" + "k8s.io/kube-state-metrics/v2/pkg/customresource" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" + metricsstore "k8s.io/kube-state-metrics/v2/pkg/metrics_store" + "k8s.io/kube-state-metrics/v2/pkg/options" +) + +// Make sure the public Builder implements the public BuilderInterface. +// New internal Builder methods should be added to the public BuilderInterface. +var _ ksmtypes.BuilderInterface = &Builder{} + +// Builder helps to build store. It follows the builder pattern +// (https://en.wikipedia.org/wiki/Builder_pattern). +type Builder struct { + internal ksmtypes.BuilderInterface +} + +// NewBuilder returns a new builder. +func NewBuilder() *Builder { + b := &Builder{ + internal: internalstore.NewBuilder(), + } + return b +} + +// WithMetrics sets the metrics property of a Builder. +func (b *Builder) WithMetrics(r prometheus.Registerer) { + b.internal.WithMetrics(r) +} + +// WithEnabledResources sets the enabledResources property of a Builder. +func (b *Builder) WithEnabledResources(c []string) error { + return b.internal.WithEnabledResources(c) +} + +// WithNamespaces sets the namespaces property of a Builder. +func (b *Builder) WithNamespaces(n options.NamespaceList) { + b.internal.WithNamespaces(n) +} + +// WithFieldSelectorFilter sets the fieldSelector property of a Builder. +func (b *Builder) WithFieldSelectorFilter(fieldSelectorFilter string) { + b.internal.WithFieldSelectorFilter(fieldSelectorFilter) +} + +// WithSharding sets the shard and totalShards property of a Builder. +func (b *Builder) WithSharding(shard int32, totalShards int) { + b.internal.WithSharding(shard, totalShards) +} + +// WithContext sets the ctx property of a Builder. +func (b *Builder) WithContext(ctx context.Context) { + b.internal.WithContext(ctx) +} + +// WithKubeClient sets the kubeClient property of a Builder. +func (b *Builder) WithKubeClient(c clientset.Interface) { + b.internal.WithKubeClient(c) +} + +// WithVPAClient sets the vpaClient property of a Builder so that the verticalpodautoscaler collector can query VPA objects. +func (b *Builder) WithVPAClient(c vpaclientset.Interface) { + b.internal.WithVPAClient(c) +} + +// WithCustomResourceClients sets the customResourceClients property of a Builder. +func (b *Builder) WithCustomResourceClients(cs map[string]interface{}) { + b.internal.WithCustomResourceClients(cs) +} + +// WithUsingAPIServerCache configures whether using APIServer cache or not. +func (b *Builder) WithUsingAPIServerCache(u bool) { + b.internal.WithUsingAPIServerCache(u) +} + +// WithFamilyGeneratorFilter configures the family generator filter which decides which +// metrics are to be exposed by the store build by the Builder. +func (b *Builder) WithFamilyGeneratorFilter(l generator.FamilyGeneratorFilter) { + b.internal.WithFamilyGeneratorFilter(l) +} + +// WithAllowAnnotations configures which annotations can be returned for metrics +func (b *Builder) WithAllowAnnotations(annotations map[string][]string) { + b.internal.WithAllowAnnotations(annotations) +} + +// WithAllowLabels configures which labels can be returned for metrics +func (b *Builder) WithAllowLabels(l map[string][]string) error { + return b.internal.WithAllowLabels(l) +} + +// WithGenerateStoresFunc configures a custom generate store function +func (b *Builder) WithGenerateStoresFunc(f ksmtypes.BuildStoresFunc) { + b.internal.WithGenerateStoresFunc(f) +} + +// WithGenerateCustomResourceStoresFunc configures a custom generate custom resource store function +func (b *Builder) WithGenerateCustomResourceStoresFunc(f ksmtypes.BuildCustomResourceStoresFunc) { + b.internal.WithGenerateCustomResourceStoresFunc(f) +} + +// DefaultGenerateStoresFunc returns default buildStore function +func (b *Builder) DefaultGenerateStoresFunc() ksmtypes.BuildStoresFunc { + return b.internal.DefaultGenerateStoresFunc() +} + +// DefaultGenerateCustomResourceStoresFunc returns default buildStores function +func (b *Builder) DefaultGenerateCustomResourceStoresFunc() ksmtypes.BuildCustomResourceStoresFunc { + return b.internal.DefaultGenerateCustomResourceStoresFunc() +} + +// WithCustomResourceStoreFactories returns configures a custom resource stores factory +func (b *Builder) WithCustomResourceStoreFactories(fs ...customresource.RegistryFactory) { + b.internal.WithCustomResourceStoreFactories(fs...) +} + +// Build initializes and registers all enabled stores. +// Returns metric writers. +func (b *Builder) Build() metricsstore.MetricsWriterList { + return b.internal.Build() +} + +// BuildStores initializes and registers all enabled stores. +// Returns metric stores. +func (b *Builder) BuildStores() [][]cache.Store { + return b.internal.BuildStores() +} diff --git a/pkg/builder/builder_test.go b/pkg/builder/builder_test.go new file mode 100644 index 0000000000..7189cb1c66 --- /dev/null +++ b/pkg/builder/builder_test.go @@ -0,0 +1,126 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder_test + +import ( + "reflect" + "testing" + + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "k8s.io/kube-state-metrics/v2/pkg/builder" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +var ( + fakeMetricLists = [][]string{ + {"metric0.1", "metric0.2"}, + {"metric1.1", "metric1.2"}, + } +) + +// BuilderInterface and Builder are public, and designed to allow +// injecting custom stores notably when ksm is used as a library. +// This test case ensures we don't break compatibility for external consumers. +func TestBuilderWithCustomStore(t *testing.T) { + b := builder.NewBuilder() + b.WithFamilyGeneratorFilter(generator.NewCompositeFamilyGeneratorFilter()) + err := b.WithEnabledResources([]string{"pods"}) + if err != nil { + t.Fatal(err) + } + + b.WithGenerateStoresFunc(customStore) + var fStores []*fakeStore + for _, stores := range b.BuildStores() { + for _, store := range stores { + fStores = append(fStores, store.(*fakeStore)) + } + } + + for i, fStore := range fStores { + metrics := fStore.List() + for j, m := range metrics { + if !reflect.DeepEqual(m, fakeMetricLists[i][j]) { + t.Fatalf("Unexpected store values: want %v found %v", fakeMetricLists[i], metrics) + } + } + } +} + +func customStore(metricFamilies []generator.FamilyGenerator, + expectedType interface{}, + listWatchFunc func(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher, + useAPIServerCache bool, +) []cache.Store { + stores := make([]cache.Store, 0, 2) + stores = append(stores, newFakeStore(fakeMetricLists[0])) + stores = append(stores, newFakeStore(fakeMetricLists[1])) + return stores +} + +func newFakeStore(metrics []string) *fakeStore { + return &fakeStore{ + metrics: metrics, + } +} + +type fakeStore struct { + metrics []string +} + +func (s *fakeStore) Add(obj interface{}) error { + return nil +} + +func (s *fakeStore) Update(obj interface{}) error { + return nil +} + +func (s *fakeStore) Delete(obj interface{}) error { + return nil +} + +func (s *fakeStore) List() []interface{} { + metrics := make([]interface{}, len(s.metrics)) + for i, m := range s.metrics { + metrics[i] = m + } + + return metrics +} + +func (s *fakeStore) ListKeys() []string { + return nil +} + +func (s *fakeStore) Get(obj interface{}) (item interface{}, exists bool, err error) { + return nil, false, nil +} + +func (s *fakeStore) GetByKey(key string) (item interface{}, exists bool, err error) { + return nil, false, nil +} + +func (s *fakeStore) Replace(list []interface{}, _ string) error { + return nil +} + +func (s *fakeStore) Resync() error { + return nil +} diff --git a/pkg/builder/types/interfaces.go b/pkg/builder/types/interfaces.go new file mode 100644 index 0000000000..c8ddf40c05 --- /dev/null +++ b/pkg/builder/types/interfaces.go @@ -0,0 +1,77 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "context" + + metricsstore "k8s.io/kube-state-metrics/v2/pkg/metrics_store" + + "github.com/prometheus/client_golang/prometheus" + vpaclientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "k8s.io/kube-state-metrics/v2/pkg/customresource" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" + "k8s.io/kube-state-metrics/v2/pkg/options" +) + +// BuilderInterface represent all methods that a Builder should implements +type BuilderInterface interface { + WithMetrics(r prometheus.Registerer) + WithEnabledResources(c []string) error + WithNamespaces(n options.NamespaceList) + WithFieldSelectorFilter(fieldSelectors string) + WithSharding(shard int32, totalShards int) + WithContext(ctx context.Context) + WithKubeClient(c clientset.Interface) + WithVPAClient(c vpaclientset.Interface) + WithCustomResourceClients(cs map[string]interface{}) + WithUsingAPIServerCache(u bool) + WithFamilyGeneratorFilter(l generator.FamilyGeneratorFilter) + WithAllowAnnotations(a map[string][]string) + WithAllowLabels(l map[string][]string) error + WithGenerateStoresFunc(f BuildStoresFunc) + WithGenerateCustomResourceStoresFunc(f BuildCustomResourceStoresFunc) + DefaultGenerateStoresFunc() BuildStoresFunc + DefaultGenerateCustomResourceStoresFunc() BuildCustomResourceStoresFunc + WithCustomResourceStoreFactories(fs ...customresource.RegistryFactory) + Build() metricsstore.MetricsWriterList + BuildStores() [][]cache.Store +} + +// BuildStoresFunc function signature that is used to return a list of cache.Store +type BuildStoresFunc func(metricFamilies []generator.FamilyGenerator, + expectedType interface{}, + listWatchFunc func(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher, + useAPIServerCache bool, +) []cache.Store + +// BuildCustomResourceStoresFunc function signature that is used to return a list of custom resource cache.Store +type BuildCustomResourceStoresFunc func(resourceName string, + metricFamilies []generator.FamilyGenerator, + expectedType interface{}, + listWatchFunc func(customResourceClient interface{}, ns string, fieldSelector string) cache.ListerWatcher, + useAPIServerCache bool, +) []cache.Store + +// AllowDenyLister interface for AllowDeny lister that can allow or exclude metrics by there names +type AllowDenyLister interface { + IsIncluded(string) bool + IsExcluded(string) bool +} diff --git a/pkg/customresource/registry_factory.go b/pkg/customresource/registry_factory.go new file mode 100644 index 0000000000..f69620248b --- /dev/null +++ b/pkg/customresource/registry_factory.go @@ -0,0 +1,118 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package customresource + +import ( + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +// RegistryFactory is a registry interface for a CustomResourceStore. +// Users who want to extend the kube-state-metrics to support Custom Resource metrics should +// implement this interface. +type RegistryFactory interface { + // Name returns the name of custom resource. + // + // Example: + // + // func (f *FooFactory) Name() string { + // return "foos" + // } + Name() string + + // CreateClient creates a new custom resource client for the given config. + // + // Example: + // + // func (f *FooFactory) CreateClient(cfg *rest.Config) (interface{}, error) { + // return clientset.NewForConfig(cfg) + // } + CreateClient(cfg *rest.Config) (interface{}, error) + + // MetricFamilyGenerators returns the metric family generators to generate metric families with a + // Kubernetes custom resource object. + // + // Example: + // + // func (f *FooFactory) MetricFamilyGenerators(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + // return []generator.FamilyGenerator{ + // *generator.NewFamilyGeneratorWithStability( + // "kube_foo_spec_replicas", + // "Number of desired replicas for a foo.", + // metric.Gauge, + // basemetrics.ALPHA, + // "", + // wrapFooFunc(func(f *samplev1alpha1.Foo) *metric.Family { + // return &metric.Family{ + // Metrics: []*metric.Metric{ + // { + // Value: float64(*f.Spec.Replicas), + // }, + // }, + // } + // }), + // ), + // *generator.NewFamilyGeneratorWithStability( + // "kube_foo_status_replicas_available", + // "The number of available replicas per foo.", + // metric.Gauge, + // basemetrics.ALPHA, + // "", + // wrapFooFunc(func(f *samplev1alpha1.Foo) *metric.Family { + // return &metric.Family{ + // Metrics: []*metric.Metric{ + // { + // Value: float64(f.Status.AvailableReplicas), + // }, + // }, + // } + // }), + // ), + // } + // } + MetricFamilyGenerators(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator + + // ExpectedType returns a pointer to an empty custom resource object. + // + // Example: + // + // func (f *FooFactory) ExpectedType() interface{} { + // return &samplev1alpha1.Foo{} + // } + ExpectedType() interface{} + + // ListWatch constructs a cache.ListerWatcher of the custom resource object. + // + // Example: + // + // func (f *FooFactory) ListWatch(customResourceClient interface{}, ns string, fieldSelector string) cache.ListerWatcher { + // client := customResourceClient.(*clientset.Clientset) + // return &cache.ListWatch{ + // ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + // opts.FieldSelector = fieldSelector + // return client.SamplecontrollerV1alpha1().Foos(ns).List(context.Background(), opts) + // }, + // WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + // opts.FieldSelector = fieldSelector + // return client.SamplecontrollerV1alpha1().Foos(ns).Watch(context.Background(), opts) + // }, + // } + // } + ListWatch(customResourceClient interface{}, ns string, fieldSelector string) cache.ListerWatcher +} diff --git a/pkg/customresourcestate/config.go b/pkg/customresourcestate/config.go new file mode 100644 index 0000000000..4ec21f0560 --- /dev/null +++ b/pkg/customresourcestate/config.go @@ -0,0 +1,181 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package customresourcestate + +import ( + "fmt" + "strings" + + "github.com/gobuffalo/flect" + "k8s.io/klog/v2" + + "k8s.io/kube-state-metrics/v2/pkg/customresource" +) + +// customResourceState is used to prefix the auto-generated GVK labels as well as an appendix for the metric itself +// if no custom metric name is defined +const customResourceState string = "customresource" + +// Metrics is the top level configuration object. +type Metrics struct { + Spec MetricsSpec `yaml:"spec" json:"spec"` +} + +// MetricsSpec is the configuration describing the custom resource state metrics to generate. +type MetricsSpec struct { + // Resources is the list of custom resources to be monitored. A resource with the same GroupVersionKind may appear + // multiple times (e.g., to customize the namespace or subsystem,) but will incur additional overhead. + Resources []Resource `yaml:"resources" json:"resources"` +} + +// Resource configures a custom resource for metric generation. +type Resource struct { + // MetricNamePrefix defines a prefix for all metrics of the resource. + // If set to "", no prefix will be added. + // Example: If set to "foo", MetricNamePrefix will be "foo_". + MetricNamePrefix *string `yaml:"metricNamePrefix" json:"metricNamePrefix"` + + // GroupVersionKind of the custom resource to be monitored. + GroupVersionKind GroupVersionKind `yaml:"groupVersionKind" json:"groupVersionKind"` + + // Labels are added to all metrics. If the same key is used in a metric, the value from the metric will overwrite the value here. + Labels `yaml:",inline" json:",inline"` + + // Metrics are the custom resource fields to be collected. + Metrics []Generator `yaml:"metrics" json:"metrics"` + // ErrorLogV defines the verbosity threshold for errors logged for this resource. + ErrorLogV klog.Level `yaml:"errorLogV" json:"errorLogV"` + + // ResourcePlural sets the plural name of the resource. Defaults to the plural version of the Kind according to flect.Pluralize. + ResourcePlural string `yaml:"resourcePlural" json:"resourcePlural"` +} + +// GetMetricNamePrefix returns the prefix to use for metrics. +func (r Resource) GetMetricNamePrefix() string { + p := r.MetricNamePrefix + if p == nil { + return "kube_" + customResourceState + } + return *p +} + +// GetResourceName returns the lowercase, plural form of the resource Kind. This is ResourcePlural if it is set. +func (r Resource) GetResourceName() string { + if r.ResourcePlural != "" { + return r.ResourcePlural + } + // kubebuilder default: + return strings.ToLower(flect.Pluralize(r.GroupVersionKind.Kind)) +} + +// GroupVersionKind is the Kubernetes group, version, and kind of a resource. +type GroupVersionKind struct { + Group string `yaml:"group" json:"group"` + Version string `yaml:"version" json:"version"` + Kind string `yaml:"kind" json:"kind"` +} + +// Labels is common configuration of labels to add to metrics. +type Labels struct { + // CommonLabels are added to all metrics. + CommonLabels map[string]string `yaml:"commonLabels" json:"commonLabels"` + // LabelsFromPath adds additional labels where the value is taken from a field in the resource. + LabelsFromPath map[string][]string `yaml:"labelsFromPath" json:"labelsFromPath"` +} + +// Merge combines the labels from two configs, returning a new config. The other Labels will overwrite keys in this Labels. +func (l Labels) Merge(other Labels) Labels { + common := make(map[string]string) + paths := make(map[string][]string) + + for k, v := range l.CommonLabels { + common[k] = v + } + for k, v := range l.LabelsFromPath { + paths[k] = v + } + for k, v := range other.CommonLabels { + common[k] = v + } + for k, v := range other.LabelsFromPath { + paths[k] = v + } + return Labels{ + CommonLabels: common, + LabelsFromPath: paths, + } +} + +// Generator describes a unique metric name. +type Generator struct { + // Name of the metric. Subject to prefixing based on the configuration of the Resource. + Name string `yaml:"name" json:"name"` + // Help text for the metric. + Help string `yaml:"help" json:"help"` + // Each targets a value or values from the resource. + Each Metric `yaml:"each" json:"each"` + + // Labels are added to all metrics. Labels from Each will overwrite these if using the same key. + Labels `yaml:",inline" json:",inline"` // json will inline because it is already tagged + // ErrorLogV defines the verbosity threshold for errors logged for this metric. Must be non-zero to override the resource setting. + ErrorLogV klog.Level `yaml:"errorLogV" json:"errorLogV"` +} + +// Metric defines a metric to expose. +// +union +type Metric struct { + // Type defines the type of the metric. + // +unionDiscriminator + Type MetricType `yaml:"type" json:"type"` + + // Gauge defines a gauge metric. + // +optional + Gauge *MetricGauge `yaml:"gauge" json:"gauge"` + // StateSet defines a state set metric. + // +optional + StateSet *MetricStateSet `yaml:"stateSet" json:"stateSet"` + // Info defines an info metric. + // +optional + Info *MetricInfo `yaml:"info" json:"info"` +} + +// ConfigDecoder is for use with FromConfig. +type ConfigDecoder interface { + Decode(v interface{}) (err error) +} + +// FromConfig decodes a configuration source into a slice of customresource.RegistryFactory that are ready to use. +func FromConfig(decoder ConfigDecoder) ([]customresource.RegistryFactory, error) { + var crconfig Metrics + var factories []customresource.RegistryFactory + factoriesIndex := map[string]bool{} + if err := decoder.Decode(&crconfig); err != nil { + return nil, fmt.Errorf("failed to parse Custom Resource State metrics: %w", err) + } + for _, resource := range crconfig.Spec.Resources { + factory, err := NewCustomResourceMetrics(resource) + if err != nil { + return nil, fmt.Errorf("failed to create metrics factory for %s: %w", resource.GroupVersionKind, err) + } + if _, ok := factoriesIndex[factory.Name()]; ok { + return nil, fmt.Errorf("found multiple custom resource configurations for the same resource %s", factory.Name()) + } + factoriesIndex[factory.Name()] = true + factories = append(factories, factory) + } + return factories, nil +} diff --git a/pkg/customresourcestate/config_metrics_types.go b/pkg/customresourcestate/config_metrics_types.go new file mode 100644 index 0000000000..6e8e9167cd --- /dev/null +++ b/pkg/customresourcestate/config_metrics_types.go @@ -0,0 +1,69 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package customresourcestate + +// MetricType is the type of a metric. +type MetricType string + +// Supported metric types. +const ( + MetricTypeGauge MetricType = "Gauge" + MetricTypeStateSet MetricType = "StateSet" + MetricTypeInfo MetricType = "Info" +) + +// MetricMeta are variables which may used for any metric type. +type MetricMeta struct { + // LabelsFromPath adds additional labels where the value of the label is taken from a field under Path. + LabelsFromPath map[string][]string `yaml:"labelsFromPath" json:"labelsFromPath"` + // Path is the path to to generate metric(s) for. + Path []string `yaml:"path" json:"path"` +} + +// MetricGauge targets a Path that may be a single value, array, or object. Arrays and objects will generate a metric per element. +// Ref: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#gauge +type MetricGauge struct { + MetricMeta `yaml:",inline" json:",inline"` + + // ValueFrom is the path to a numeric field under Path that will be the metric value. + ValueFrom []string `yaml:"valueFrom" json:"valueFrom"` + // LabelFromKey adds a label with the given name if Path is an object. The label value will be the object key. + LabelFromKey string `yaml:"labelFromKey" json:"labelFromKey"` + // NilIsZero indicates that if a value is nil it will be treated as zero value. + NilIsZero bool `yaml:"nilIsZero" json:"nilIsZero"` +} + +// MetricInfo is a metric which is used to expose textual information. +// Ref: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#info +type MetricInfo struct { + MetricMeta `yaml:",inline" json:",inline"` + // LabelFromKey adds a label with the given name if Path is an object. The label value will be the object key. + LabelFromKey string `yaml:"labelFromKey" json:"labelFromKey"` +} + +// MetricStateSet is a metric which represent a series of related boolean values, also called a bitset. +// Ref: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#stateset +type MetricStateSet struct { + MetricMeta `yaml:",inline" json:",inline"` + + // List is the list of values to expose a value for. + List []string `yaml:"list" json:"list"` + // LabelName is the key of the label which is used for each entry in List to expose the value. + LabelName string `yaml:"labelName" json:"labelName"` + // ValueFrom is the subpath to compare the list to. + ValueFrom []string `yaml:"valueFrom" json:"valueFrom"` +} diff --git a/pkg/customresourcestate/config_test.go b/pkg/customresourcestate/config_test.go new file mode 100644 index 0000000000..7ae11985f2 --- /dev/null +++ b/pkg/customresourcestate/config_test.go @@ -0,0 +1,63 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package customresourcestate + +import ( + _ "embed" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" + "k8s.io/klog/v2" +) + +//go:embed example_config.yaml +var testData string + +func Test_Metrics_deserialization(t *testing.T) { + var m Metrics + assert.NoError(t, yaml.NewDecoder(strings.NewReader(testData)).Decode(&m)) + assert.Equal(t, "active_count", m.Spec.Resources[0].Metrics[0].Name) + + t.Run("can create resource factory", func(t *testing.T) { + rf, err := NewCustomResourceMetrics(m.Spec.Resources[0]) + assert.NoError(t, err) + + t.Run("labels are merged", func(t *testing.T) { + assert.Equal(t, map[string]string{ + "name": mustCompilePath(t, "metadata", "name").String(), + }, toPaths(rf.(*customResourceMetrics).Families[1].LabelFromPath)) + }) + + t.Run("errorLogV", func(t *testing.T) { + assert.Equal(t, klog.Level(5), rf.(*customResourceMetrics).Families[1].ErrorLogV) + }) + + t.Run("resource name", func(t *testing.T) { + assert.Equal(t, rf.(*customResourceMetrics).ResourceName, "foos") + }) + }) +} + +func toPaths(m map[string]valuePath) map[string]string { + out := make(map[string]string) + for k, v := range m { + out[k] = v.String() + } + return out +} diff --git a/pkg/customresourcestate/custom_resource_metrics.go b/pkg/customresourcestate/custom_resource_metrics.go new file mode 100644 index 0000000000..da45a7533a --- /dev/null +++ b/pkg/customresourcestate/custom_resource_metrics.go @@ -0,0 +1,113 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package customresourcestate + +import ( + "context" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + + "k8s.io/kube-state-metrics/v2/pkg/customresource" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +// customResourceMetrics is an implementation of the customresource.RegistryFactory +// interface which provides metrics for custom resources defined in a configuration file. +type customResourceMetrics struct { + MetricNamePrefix string + GroupVersionKind schema.GroupVersionKind + ResourceName string + Families []compiledFamily +} + +var _ customresource.RegistryFactory = &customResourceMetrics{} + +// NewCustomResourceMetrics creates a customresource.RegistryFactory from a configuration object. +func NewCustomResourceMetrics(resource Resource) (customresource.RegistryFactory, error) { + compiled, err := compile(resource) + if err != nil { + return nil, err + } + gvk := schema.GroupVersionKind(resource.GroupVersionKind) + return &customResourceMetrics{ + MetricNamePrefix: resource.GetMetricNamePrefix(), + GroupVersionKind: gvk, + Families: compiled, + ResourceName: resource.GetResourceName(), + }, nil +} + +func (s customResourceMetrics) Name() string { + return s.ResourceName +} + +func (s customResourceMetrics) CreateClient(cfg *rest.Config) (interface{}, error) { + c, err := dynamic.NewForConfig(cfg) + if err != nil { + return nil, err + } + return c.Resource(schema.GroupVersionResource{ + Group: s.GroupVersionKind.Group, + Version: s.GroupVersionKind.Version, + Resource: s.ResourceName, + }), nil +} + +func (s customResourceMetrics) MetricFamilyGenerators(_, _ []string) (result []generator.FamilyGenerator) { + klog.InfoS("Custom resource state added metrics", "familyNames", s.names()) + for _, f := range s.Families { + result = append(result, famGen(f)) + } + + return result +} + +func (s customResourceMetrics) ExpectedType() interface{} { + u := unstructured.Unstructured{} + u.SetGroupVersionKind(s.GroupVersionKind) + return &u +} + +func (s customResourceMetrics) ListWatch(customResourceClient interface{}, ns string, fieldSelector string) cache.ListerWatcher { + api := customResourceClient.(dynamic.NamespaceableResourceInterface).Namespace(ns) + ctx := context.Background() + return &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + options.FieldSelector = fieldSelector + return api.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + options.FieldSelector = fieldSelector + return api.Watch(ctx, options) + }, + } +} + +func (s customResourceMetrics) names() (names []string) { + for _, family := range s.Families { + names = append(names, family.Name) + } + return names +} diff --git a/pkg/customresourcestate/custom_resource_metrics_test.go b/pkg/customresourcestate/custom_resource_metrics_test.go new file mode 100644 index 0000000000..e62e333c1c --- /dev/null +++ b/pkg/customresourcestate/custom_resource_metrics_test.go @@ -0,0 +1,248 @@ +/* +Copyright 2022 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package customresourcestate + +import ( + "encoding/json" + "reflect" + "testing" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/utils/pointer" +) + +func TestNewCustomResourceMetrics(t *testing.T) { + + tests := []struct { + r Resource + wantErr bool + wantResult *customResourceMetrics + name string + }{ + { + // https://github.com/kubernetes/kube-state-metrics/issues/1886 + name: "cr metric with dynamic metric type", + r: Resource{ + GroupVersionKind: GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + }, + Labels: Labels{ + LabelsFromPath: map[string][]string{ + "name": {"metadata", "name"}, + }, + CommonLabels: map[string]string{ + "hello": "world", + }, + }, + Metrics: []Generator{ + { + Name: "test_metrics", + Help: "metrics for testing", + Each: Metric{ + Type: MetricTypeInfo, + Info: &MetricInfo{ + MetricMeta: MetricMeta{ + Path: []string{ + "metadata", + "annotations", + }, + }, + LabelFromKey: "test", + }, + }, + }, + }, + }, + wantErr: false, + wantResult: &customResourceMetrics{ + MetricNamePrefix: "kube_customresource", + GroupVersionKind: schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + }, + ResourceName: "deployments", + Families: []compiledFamily{ + { + Name: "kube_customresource_test_metrics", + Help: "metrics for testing", + Each: &compiledInfo{}, + Labels: map[string]string{ + "customresource_group": "apps", + "customresource_kind": "Deployment", + "customresource_version": "v1", + "hello": "world", + }, + LabelFromPath: map[string]valuePath{ + "name": mustCompilePath(t, "metadata", "name"), + }, + }, + }, + }, + }, + { + name: "cr metric with custom prefix", + r: Resource{ + GroupVersionKind: GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + }, + Labels: Labels{ + LabelsFromPath: map[string][]string{ + "name": {"metadata", "name"}, + }, + CommonLabels: map[string]string{ + "hello": "world", + }, + }, + Metrics: []Generator{ + { + Name: "test_metrics", + Help: "metrics for testing", + Each: Metric{ + Type: MetricTypeInfo, + Info: &MetricInfo{ + MetricMeta: MetricMeta{ + Path: []string{ + "metadata", + "annotations", + }, + }, + LabelFromKey: "test", + }, + }, + }, + }, + MetricNamePrefix: pointer.String("apps_deployment"), + }, + wantErr: false, + wantResult: &customResourceMetrics{ + MetricNamePrefix: "apps_deployment", + GroupVersionKind: schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + }, + ResourceName: "deployments", + Families: []compiledFamily{ + { + Name: "apps_deployment_test_metrics", + Help: "metrics for testing", + Each: &compiledInfo{}, + Labels: map[string]string{ + "customresource_group": "apps", + "customresource_kind": "Deployment", + "customresource_version": "v1", + "hello": "world", + }, + LabelFromPath: map[string]valuePath{ + "name": mustCompilePath(t, "metadata", "name"), + }, + }, + }, + }, + }, + { + name: "cr metric with custom prefix - expect error", + r: Resource{ + GroupVersionKind: GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + }, + Labels: Labels{ + LabelsFromPath: map[string][]string{ + "name": {"metadata", "name"}, + }, + CommonLabels: map[string]string{ + "hello": "world", + }, + }, + Metrics: []Generator{ + { + Name: "test_metrics", + Help: "metrics for testing", + Each: Metric{ + Type: MetricTypeInfo, + Info: &MetricInfo{ + MetricMeta: MetricMeta{ + Path: []string{ + "metadata", + "annotations", + }, + }, + LabelFromKey: "test", + }, + }, + }, + }, + MetricNamePrefix: pointer.String("apps_deployment"), + }, + wantErr: true, + wantResult: &customResourceMetrics{ + GroupVersionKind: schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + }, + ResourceName: "deployments", + Families: []compiledFamily{ + { + Name: "apps_deployment_test_metrics", + Help: "metrics for testing", + Each: &compiledInfo{}, + Labels: map[string]string{ + "customresource_group": "apps", + "customresource_kind": "Deployment", + "customresource_version": "v1", + "hello": "world", + }, + LabelFromPath: map[string]valuePath{ + "name": mustCompilePath(t, "metadata", "name"), + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v, err := NewCustomResourceMetrics(tt.r) + if err != nil { + t.Errorf(err.Error()) + } + + // convert to JSON for easier nil comparison + ttWantJSON, _ := json.Marshal(tt.wantResult) + customResourceMetricJSON, _ := json.Marshal(v.(*customResourceMetrics)) + + if !tt.wantErr && !reflect.DeepEqual(ttWantJSON, customResourceMetricJSON) { + t.Errorf("NewCustomResourceMetrics: error expected %v", tt.wantErr) + t.Errorf("NewCustomResourceMetrics:\n %#v\n is not deep equal\n %#v", v, tt.wantResult) + } + + if tt.wantErr && reflect.DeepEqual(ttWantJSON, customResourceMetricJSON) { + t.Errorf("NewCustomResourceMetrics: error expected %v", tt.wantErr) + t.Errorf("NewCustomResourceMetrics:\n %#v\n is not deep equal\n %#v", v, tt.wantResult) + } + }) + } +} diff --git a/pkg/customresourcestate/example_config.yaml b/pkg/customresourcestate/example_config.yaml new file mode 100644 index 0000000000..39a479b2be --- /dev/null +++ b/pkg/customresourcestate/example_config.yaml @@ -0,0 +1,64 @@ +# Copyright 2021 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: CustomResourceStateMetrics +spec: + resources: + - + groupVersionKind: + group: myteam.io + kind: "Foo" + version: "v1" + labelsFromPath: + name: [ metadata, name ] + metrics: + - name: "active_count" + help: "Number Foo Bars active" + each: + type: Gauge + gauge: + path: [status, active] + labelFromKey: type + labelsFromPath: + bar: [bar] + value: [count] + commonLabels: + custom_metric: "yes" + + labelsFromPath: + "*": [metadata, labels] # copy all labels from CR labels + foo: [metadata, labels, foo] # copy a single label (overrides *) + + - name: "other_count" + each: + type: Gauge + gauge: + path: [status, other] + errorLogV: 5 + + - name: "info" + each: + type: Info + info: + path: [spec, version] + errorLogV: 5 + + - name: "phase" + each: + type: StateSet + stateSet: + path: [status, phase] + labelName: phase + list: [Active, Running, Terminating] + errorLogV: 5 diff --git a/pkg/customresourcestate/registry_factory.go b/pkg/customresourcestate/registry_factory.go new file mode 100644 index 0000000000..9b4674d9de --- /dev/null +++ b/pkg/customresourcestate/registry_factory.go @@ -0,0 +1,724 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package customresourcestate + +import ( + "errors" + "fmt" + "math" + "sort" + "strconv" + "strings" + "time" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/klog/v2" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +func compile(resource Resource) ([]compiledFamily, error) { + var families []compiledFamily + // Explicitly add GVK labels to all CR metrics. + if resource.CommonLabels == nil { + resource.CommonLabels = map[string]string{} + } + resource.CommonLabels[customResourceState+"_group"] = resource.GroupVersionKind.Group + resource.CommonLabels[customResourceState+"_version"] = resource.GroupVersionKind.Version + resource.CommonLabels[customResourceState+"_kind"] = resource.GroupVersionKind.Kind + for _, f := range resource.Metrics { + family, err := compileFamily(f, resource) + if err != nil { + return nil, fmt.Errorf("%s: %w", f.Name, err) + } + families = append(families, *family) + } + return families, nil +} + +func compileCommon(c MetricMeta) (*compiledCommon, error) { + eachPath, err := compilePath(c.Path) + if err != nil { + return nil, fmt.Errorf("path: %w", err) + } + eachLabelsFromPath, err := compilePaths(c.LabelsFromPath) + if err != nil { + return nil, fmt.Errorf("labelsFromPath: %w", err) + } + return &compiledCommon{ + path: eachPath, + labelFromPath: eachLabelsFromPath, + }, nil +} + +func compileFamily(f Generator, resource Resource) (*compiledFamily, error) { + labels := resource.Labels.Merge(f.Labels) + + metric, err := newCompiledMetric(f.Each) + if err != nil { + return nil, fmt.Errorf("compiling metric: %w", err) + } + + labelsFromPath, err := compilePaths(labels.LabelsFromPath) + if err != nil { + return nil, fmt.Errorf("labelsFromPath: %w", err) + } + + errorLogV := f.ErrorLogV + if errorLogV == 0 { + errorLogV = resource.ErrorLogV + } + return &compiledFamily{ + Name: fullName(resource, f), + ErrorLogV: errorLogV, + Help: f.Help, + Each: metric, + Labels: labels.CommonLabels, + LabelFromPath: labelsFromPath, + }, nil +} + +func fullName(resource Resource, f Generator) string { + var parts []string + if resource.GetMetricNamePrefix() != "" { + parts = append(parts, resource.GetMetricNamePrefix()) + } + parts = append(parts, f.Name) + return strings.Join(parts, "_") +} + +func compilePaths(paths map[string][]string) (result map[string]valuePath, err error) { + result = make(map[string]valuePath) + for k, v := range paths { + result[k], err = compilePath(v) + if err != nil { + return nil, fmt.Errorf("%s: %w", k, err) + } + } + return result, nil +} + +type compiledEach compiledMetric + +type compiledCommon struct { + labelFromPath map[string]valuePath + path valuePath + t metric.Type +} + +func (c compiledCommon) Path() valuePath { + return c.path +} +func (c compiledCommon) LabelFromPath() map[string]valuePath { + return c.labelFromPath +} +func (c compiledCommon) Type() metric.Type { + return c.t +} + +type eachValue struct { + Labels map[string]string + Value float64 +} + +type compiledMetric interface { + Values(v interface{}) (result []eachValue, err []error) + Path() valuePath + LabelFromPath() map[string]valuePath + Type() metric.Type +} + +// newCompiledMetric returns a compiledMetric depending on the given metric type. +func newCompiledMetric(m Metric) (compiledMetric, error) { + switch m.Type { + case MetricTypeGauge: + if m.Gauge == nil { + return nil, errors.New("expected each.gauge to not be nil") + } + cc, err := compileCommon(m.Gauge.MetricMeta) + cc.t = metric.Gauge + if err != nil { + return nil, fmt.Errorf("each.gauge: %w", err) + } + valueFromPath, err := compilePath(m.Gauge.ValueFrom) + if err != nil { + return nil, fmt.Errorf("each.gauge.valueFrom: %w", err) + } + return &compiledGauge{ + compiledCommon: *cc, + ValueFrom: valueFromPath, + NilIsZero: m.Gauge.NilIsZero, + labelFromKey: m.Gauge.LabelFromKey, + }, nil + case MetricTypeInfo: + if m.Info == nil { + return nil, errors.New("expected each.info to not be nil") + } + cc, err := compileCommon(m.Info.MetricMeta) + cc.t = metric.Info + if err != nil { + return nil, fmt.Errorf("each.info: %w", err) + } + return &compiledInfo{ + compiledCommon: *cc, + labelFromKey: m.Info.LabelFromKey, + }, nil + case MetricTypeStateSet: + if m.StateSet == nil { + return nil, errors.New("expected each.stateSet to not be nil") + } + cc, err := compileCommon(m.StateSet.MetricMeta) + cc.t = metric.StateSet + if err != nil { + return nil, fmt.Errorf("each.stateSet: %w", err) + } + valueFromPath, err := compilePath(m.StateSet.ValueFrom) + if err != nil { + return nil, fmt.Errorf("each.gauge.valueFrom: %w", err) + } + return &compiledStateSet{ + compiledCommon: *cc, + List: m.StateSet.List, + LabelName: m.StateSet.LabelName, + ValueFrom: valueFromPath, + }, nil + default: + return nil, fmt.Errorf("unknown metric type %s", m.Type) + } +} + +type compiledGauge struct { + compiledCommon + ValueFrom valuePath + NilIsZero bool + labelFromKey string +} + +func (c *compiledGauge) Values(v interface{}) (result []eachValue, errs []error) { + onError := func(err error) { + errs = append(errs, fmt.Errorf("%s: %v", c.Path(), err)) + } + + switch iter := v.(type) { + case map[string]interface{}: + for key, it := range iter { + // TODO: Handle multi-length valueFrom paths (https://github.com/kubernetes/kube-state-metrics/pull/1958#discussion_r1099243161). + // Try to deduce `valueFrom`'s value from the current element. + var ev *eachValue + var err error + var didResolveValueFrom bool + // `valueFrom` will ultimately be rendered into a string and sent to the fallback in place, which also expects a string. + // So we'll do the same and operate on the string representation of `valueFrom`'s value. + sValueFrom := c.ValueFrom.String() + // No comma means we're looking at a unit-length path (in an array). + if !strings.Contains(sValueFrom, ",") && + sValueFrom[0] == '[' && sValueFrom[len(sValueFrom)-1] == ']' && + // "[...]" and not "[]". + len(sValueFrom) > 2 { + extractedValueFrom := sValueFrom[1 : len(sValueFrom)-1] + if key == extractedValueFrom { + gotFloat, err := toFloat64(it, c.NilIsZero) + if err != nil { + onError(fmt.Errorf("[%s]: %w", key, err)) + continue + } + labels := make(map[string]string) + ev = &eachValue{ + Labels: labels, + Value: gotFloat, + } + didResolveValueFrom = true + } + } + // Fallback to the regular path resolution, if we didn't manage to resolve `valueFrom`'s value. + if !didResolveValueFrom { + ev, err = c.value(it) + if ev == nil { + continue + } + } + if err != nil { + onError(fmt.Errorf("[%s]: %w", key, err)) + continue + } + if _, ok := ev.Labels[c.labelFromKey]; ok { + onError(fmt.Errorf("labelFromKey (%s) generated labels conflict with labelsFromPath, consider renaming it", c.labelFromKey)) + continue + } + if key != "" && c.labelFromKey != "" { + ev.Labels[c.labelFromKey] = key + } + addPathLabels(it, c.LabelFromPath(), ev.Labels) + result = append(result, *ev) + } + case []interface{}: + for i, it := range iter { + value, err := c.value(it) + if err != nil { + onError(fmt.Errorf("[%d]: %w", i, err)) + continue + } + addPathLabels(it, c.LabelFromPath(), value.Labels) + result = append(result, *value) + } + default: + value, err := c.value(v) + if err != nil { + onError(err) + break + } + addPathLabels(v, c.LabelFromPath(), value.Labels) + result = append(result, *value) + } + return +} + +type compiledInfo struct { + compiledCommon + labelFromKey string +} + +func (c *compiledInfo) Values(v interface{}) (result []eachValue, errs []error) { + onError := func(err ...error) { + errs = append(errs, fmt.Errorf("%s: %v", c.Path(), err)) + } + + switch iter := v.(type) { + case []interface{}: + for _, obj := range iter { + ev, err := c.values(obj) + if len(err) > 0 { + onError(err...) + continue + } + result = append(result, ev...) + } + case map[string]interface{}: + value, err := c.values(v) + if err != nil { + onError(err...) + break + } + for _, ev := range value { + if _, ok := ev.Labels[c.labelFromKey]; ok { + onError(fmt.Errorf("labelFromKey (%s) generated labels conflict with labelsFromPath, consider renaming it", c.labelFromKey)) + continue + } + } + // labelFromKey logic + for key := range iter { + if key != "" && c.labelFromKey != "" { + result = append(result, eachValue{ + Labels: map[string]string{ + c.labelFromKey: key, + }, + Value: 1, + }) + } + } + result = append(result, value...) + default: + result, errs = c.values(v) + } + + return +} + +func (c *compiledInfo) values(v interface{}) (result []eachValue, err []error) { + if v == nil { + return + } + value := eachValue{Value: 1, Labels: map[string]string{}} + addPathLabels(v, c.labelFromPath, value.Labels) + if len(value.Labels) != 0 { + result = append(result, value) + } + return +} + +type compiledStateSet struct { + compiledCommon + ValueFrom valuePath + List []string + LabelName string +} + +func (c *compiledStateSet) Values(v interface{}) (result []eachValue, errs []error) { + if vs, isArray := v.([]interface{}); isArray { + for _, obj := range vs { + ev, err := c.values(obj) + if len(err) > 0 { + errs = append(errs, err...) + continue + } + result = append(result, ev...) + } + return + } + + return c.values(v) +} + +func (c *compiledStateSet) values(v interface{}) (result []eachValue, errs []error) { + comparable := c.ValueFrom.Get(v) + value, ok := comparable.(string) + if !ok { + return []eachValue{}, []error{fmt.Errorf("%s: expected value for path to be string, got %T", c.path, comparable)} + } + + for _, entry := range c.List { + ev := eachValue{Value: 0, Labels: map[string]string{}} + if value == entry { + ev.Value = 1 + } + ev.Labels[c.LabelName] = entry + addPathLabels(v, c.labelFromPath, ev.Labels) + result = append(result, ev) + } + return +} + +// less compares two maps of labels by keys and values +func less(a, b map[string]string) bool { + var aKeys, bKeys sort.StringSlice + for k := range a { + aKeys = append(aKeys, k) + } + for k := range b { + bKeys = append(bKeys, k) + } + aKeys.Sort() + bKeys.Sort() + for i := 0; i < int(math.Min(float64(len(aKeys)), float64(len(bKeys)))); i++ { + if aKeys[i] != bKeys[i] { + return aKeys[i] < bKeys[i] + } + + va := a[aKeys[i]] + vb := b[bKeys[i]] + if va == vb { + continue + } + return va < vb + } + return len(aKeys) < len(bKeys) +} + +func (c compiledGauge) value(it interface{}) (*eachValue, error) { + labels := make(map[string]string) + got := c.ValueFrom.Get(it) + // If `valueFrom` was not resolved, respect `NilIsZero` and return. + if got == nil { + if c.NilIsZero { + return &eachValue{ + Labels: labels, + Value: 0, + }, nil + } + // no it means no iterables were passed down, meaning that the path resolution never happened + if it == nil { + return nil, fmt.Errorf("got nil while resolving path") + } + // Don't error if there was not a type-casting issue (`toFloat64`). + return nil, nil + } + value, err := toFloat64(got, c.NilIsZero) + if err != nil { + return nil, fmt.Errorf("%s: %w", c.ValueFrom, err) + } + return &eachValue{ + Labels: labels, + Value: value, + }, nil +} + +func (e eachValue) DefaultLabels(defaults map[string]string) { + for k, v := range defaults { + if _, ok := e.Labels[k]; !ok { + e.Labels[k] = v + } + } +} +func (e eachValue) ToMetric() *metric.Metric { + var keys, values []string + for k := range e.Labels { + keys = append(keys, k) + } + // make it deterministic + sort.Strings(keys) + for _, key := range keys { + values = append(values, e.Labels[key]) + } + return &metric.Metric{ + LabelKeys: keys, + LabelValues: values, + Value: e.Value, + } +} + +type compiledFamily struct { + Name string + Help string + Each compiledEach + Labels map[string]string + LabelFromPath map[string]valuePath + ErrorLogV klog.Level +} + +func (f compiledFamily) BaseLabels(obj map[string]interface{}) map[string]string { + result := make(map[string]string) + for k, v := range f.Labels { + result[k] = v + } + addPathLabels(obj, f.LabelFromPath, result) + return result +} + +func addPathLabels(obj interface{}, labels map[string]valuePath, result map[string]string) { + // *prefixed is a special case, it means copy an object + // always do that first so other labels can override + var stars []string + for k := range labels { + if strings.HasPrefix(k, "*") { + stars = append(stars, k) + } + } + sort.Strings(stars) + for _, k := range stars { + m := labels[k].Get(obj) + if kv, ok := m.(map[string]interface{}); ok { + for k, v := range kv { + result[k] = fmt.Sprintf("%v", v) + } + } + } + for k, v := range labels { + if strings.HasPrefix(k, "*") { + continue + } + value := v.Get(obj) + // skip label if value is nil + if value == nil { + continue + } + result[k] = fmt.Sprintf("%v", value) + } +} + +type pathOp struct { + part string + op func(interface{}) interface{} +} + +type valuePath []pathOp + +func (p valuePath) Get(obj interface{}) interface{} { + for _, op := range p { + if obj == nil { + return nil + } + obj = op.op(obj) + } + return obj +} + +func (p valuePath) String() string { + var b strings.Builder + b.WriteRune('[') + for i, op := range p { + if i > 0 { + b.WriteRune(',') + } + b.WriteString(op.part) + } + b.WriteRune(']') + return b.String() +} + +func compilePath(path []string) (out valuePath, _ error) { + for i := range path { + part := path[i] + if strings.HasPrefix(part, "[") && strings.HasSuffix(part, "]") { + // list lookup: [key=value] + eq := strings.SplitN(part[1:len(part)-1], "=", 2) + if len(eq) != 2 { + return nil, fmt.Errorf("invalid list lookup: %s", part) + } + key, val := eq[0], eq[1] + num, notNum := toFloat64(val, false) + boolVal, notBool := strconv.ParseBool(val) + out = append(out, pathOp{ + part: part, + op: func(m interface{}) interface{} { + if s, ok := m.([]interface{}); ok { + for _, v := range s { + if m, ok := v.(map[string]interface{}); ok { + candidate, set := m[key] + if !set { + continue + } + + if candidate == val { + return m + } + + if notNum == nil { + if i, err := toFloat64(candidate, false); err == nil && num == i { + return m + } + } + + if notBool == nil { + if v, ok := candidate.(bool); ok && v == boolVal { + return m + } + } + + } + } + } + return nil + }, + }) + } else { + out = append(out, pathOp{ + part: part, + op: func(m interface{}) interface{} { + if mp, ok := m.(map[string]interface{}); ok { + return mp[part] + } else if s, ok := m.([]interface{}); ok { + i, err := strconv.Atoi(part) + if err != nil { + // This means we are here: [ , , ... ] (eg., [ "foo", "0", ... ], i.e., .foo[0]... + // ^ + // Skip over. + return nil + } + if i < 0 { + // negative index + i += len(s) + } + if !(0 <= i && i < len(s)) { + return fmt.Errorf("list index out of range: %s", part) + } + return s[i] + } + return nil + }, + }) + } + } + return out, nil +} + +func famGen(f compiledFamily) generator.FamilyGenerator { + errLog := klog.V(f.ErrorLogV) + return generator.FamilyGenerator{ + Name: f.Name, + Type: f.Each.Type(), + Help: f.Help, + GenerateFunc: func(obj interface{}) *metric.Family { + return generate(obj.(*unstructured.Unstructured), f, errLog) + }, + } +} + +// generate generates the metrics for a custom resource. +func generate(u *unstructured.Unstructured, f compiledFamily, errLog klog.Verbose) *metric.Family { + klog.V(10).InfoS("Checked", "compiledFamilyName", f.Name, "unstructuredName", u.GetName()) + var metrics []*metric.Metric + baseLabels := f.BaseLabels(u.Object) + + values, errors := scrapeValuesFor(f.Each, u.Object) + for _, err := range errors { + errLog.ErrorS(err, f.Name) + } + + for _, v := range values { + v.DefaultLabels(baseLabels) + metrics = append(metrics, v.ToMetric()) + } + klog.V(10).InfoS("Produced metrics for", "compiledFamilyName", f.Name, "metricsLength", len(metrics), "unstructuredName", u.GetName()) + + return &metric.Family{ + Metrics: metrics, + } +} + +func scrapeValuesFor(e compiledEach, obj map[string]interface{}) ([]eachValue, []error) { + v := e.Path().Get(obj) + result, errs := e.Values(v) + + // return results in a consistent order (simplifies testing) + sort.Slice(result, func(i, j int) bool { + return less(result[i].Labels, result[j].Labels) + }) + return result, errs +} + +// toFloat64 converts the value to a float64 which is the value type for any metric. +func toFloat64(value interface{}, nilIsZero bool) (float64, error) { + var v float64 + // same as bool==false but for bool pointers + if value == nil { + if nilIsZero { + return 0, nil + } + return 0, fmt.Errorf("expected number but found nil value") + } + switch vv := value.(type) { + case bool: + if vv { + return 1, nil + } + return 0, nil + case string: + normalized := strings.ToLower(value.(string)) + if normalized == "true" || normalized == "yes" { + return 1, nil + } + if normalized == "false" || normalized == "no" { + return 0, nil + } + if t, e := time.Parse(time.RFC3339, value.(string)); e == nil { + return float64(t.Unix()), nil + } + return strconv.ParseFloat(value.(string), 64) + case byte: + v = float64(vv) + case int: + v = float64(vv) + case int32: + v = float64(vv) + case int64: + v = float64(vv) + case uint: + v = float64(vv) + case uint32: + v = float64(vv) + case uint64: + v = float64(vv) + case float32: + v = float64(vv) + case float64: + v = vv + default: + return 0, fmt.Errorf("expected number but was %v", value) + } + return v, nil +} diff --git a/pkg/customresourcestate/registry_factory_test.go b/pkg/customresourcestate/registry_factory_test.go new file mode 100644 index 0000000000..c1d6f1621c --- /dev/null +++ b/pkg/customresourcestate/registry_factory_test.go @@ -0,0 +1,480 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package customresourcestate + +import ( + "encoding/json" + "errors" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/utils/pointer" + + "k8s.io/kube-state-metrics/v2/pkg/metric" +) + +var cr map[string]interface{} + +func init() { + type Obj map[string]interface{} + type Array []interface{} + bytes, err := json.Marshal(Obj{ + "spec": Obj{ + "replicas": 1, + "version": "v0.0.0", + "order": Array{ + Obj{ + "id": 1, + "value": true, + }, + Obj{ + "id": 3, + "value": false, + }, + }, + }, + "status": Obj{ + "active": Obj{ + "type-a": 1, + "type-b": 3, + }, + "phase": "foo", + "sub": Obj{ + "type-a": Obj{ + "active": 1, + "ready": 2, + }, + "type-b": Obj{ + "active": 3, + "ready": 4, + }, + }, + "uptime": 43.21, + "condition_values": Array{ + Obj{ + "name": "a", + "value": 45, + }, + Obj{ + "name": "b", + "value": 66, + }, + }, + "conditions": Array{ + Obj{ + "type": "Ready", + "status": "True", + }, + Obj{ + "type": "Provisioned", + "status": "False", + }, + }, + }, + "metadata": Obj{ + "name": "foo", + "labels": Obj{ + "foo": "bar", + }, + "annotations": Obj{ + "qux": "quxx", + "bar": "baz", + }, + "creationTimestamp": "2022-06-28T00:00:00Z", + }, + }) + if err != nil { + panic(err) + } + _ = json.Unmarshal(bytes, &cr) +} + +func Test_addPathLabels(t *testing.T) { + type args struct { + obj interface{} + labels map[string]valuePath + want map[string]string + } + tests := []struct { + name string + args args + }{ + {name: "all", args: args{ + obj: cr, + labels: map[string]valuePath{ + "bool": mustCompilePath(t, "spec", "order", "-1", "value"), + "number": mustCompilePath(t, "spec", "replicas"), + "string": mustCompilePath(t, "metadata", "labels", "foo"), + }, + want: map[string]string{ + "bool": "false", + "number": "1", + "string": "bar", + }, + }}, + {name: "*", args: args{ + obj: cr, + labels: map[string]valuePath{ + "*1": mustCompilePath(t, "metadata", "annotations"), + "bar": mustCompilePath(t, "metadata", "labels", "foo"), + }, + want: map[string]string{ + "qux": "quxx", + "bar": "bar", + }, + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := make(map[string]string) + addPathLabels(tt.args.obj, tt.args.labels, m) + assert.Equal(t, tt.args.want, m) + }) + } +} + +func Test_values(t *testing.T) { + type tc struct { + name string + each compiledEach + wantResult []eachValue + wantErrors []error + } + + tests := []tc{ + {name: "single", each: &compiledGauge{ + compiledCommon: compiledCommon{ + path: mustCompilePath(t, "spec", "replicas"), + }, + }, wantResult: []eachValue{newEachValue(t, 1)}}, + {name: "obj", each: &compiledGauge{ + compiledCommon: compiledCommon{ + path: mustCompilePath(t, "status", "active"), + }, + labelFromKey: "type", + }, wantResult: []eachValue{ + newEachValue(t, 1, "type", "type-a"), + newEachValue(t, 3, "type", "type-b"), + }}, + {name: "deep obj", each: &compiledGauge{ + compiledCommon: compiledCommon{ + path: mustCompilePath(t, "status", "sub"), + labelFromPath: map[string]valuePath{ + "active": mustCompilePath(t, "active"), + }, + }, + labelFromKey: "type", + ValueFrom: mustCompilePath(t, "ready"), + }, wantResult: []eachValue{ + newEachValue(t, 2, "type", "type-a", "active", "1"), + newEachValue(t, 4, "type", "type-b", "active", "3"), + }}, + {name: "path-relative valueFrom value", each: &compiledGauge{ + compiledCommon: compiledCommon{ + path: mustCompilePath(t, "metadata"), + labelFromPath: map[string]valuePath{ + "name": mustCompilePath(t, "name"), + }, + }, + ValueFrom: mustCompilePath(t, "creationTimestamp"), + }, wantResult: []eachValue{ + newEachValue(t, 1.6563744e+09), + }}, + {name: "non-existent path", each: &compiledGauge{ + compiledCommon: compiledCommon{ + path: mustCompilePath(t, "foo"), + labelFromPath: map[string]valuePath{ + "name": mustCompilePath(t, "name"), + }, + }, + ValueFrom: mustCompilePath(t, "creationTimestamp"), + }, wantResult: nil, wantErrors: []error{ + errors.New("[foo]: got nil while resolving path"), + }}, + {name: "array", each: &compiledGauge{ + compiledCommon: compiledCommon{ + path: mustCompilePath(t, "status", "condition_values"), + labelFromPath: map[string]valuePath{ + "name": mustCompilePath(t, "name"), + }, + }, + ValueFrom: mustCompilePath(t, "value"), + }, wantResult: []eachValue{ + newEachValue(t, 45, "name", "a"), + newEachValue(t, 66, "name", "b"), + }}, + {name: "timestamp", each: &compiledGauge{ + compiledCommon: compiledCommon{ + path: mustCompilePath(t, "metadata", "creationTimestamp"), + }, + }, wantResult: []eachValue{ + newEachValue(t, 1656374400), + }}, + {name: "boolean_string", each: &compiledGauge{ + compiledCommon: compiledCommon{ + path: mustCompilePath(t, "spec", "paused"), + }, + NilIsZero: true, + }, wantResult: []eachValue{ + newEachValue(t, 0), + }}, + {name: "info", each: &compiledInfo{ + compiledCommon: compiledCommon{ + labelFromPath: map[string]valuePath{ + "version": mustCompilePath(t, "spec", "version"), + }, + }, + }, wantResult: []eachValue{ + newEachValue(t, 1, "version", "v0.0.0"), + }}, + {name: "info nil path", each: &compiledInfo{ + compiledCommon: compiledCommon{ + path: mustCompilePath(t, "does", "not", "exist"), + }, + }, wantResult: nil}, + {name: "info label from key", each: &compiledInfo{ + compiledCommon: compiledCommon{ + path: mustCompilePath(t, "status", "active"), + }, + labelFromKey: "type", + }, wantResult: []eachValue{ + newEachValue(t, 1, "type", "type-a"), + newEachValue(t, 1, "type", "type-b"), + }}, + {name: "stateset", each: &compiledStateSet{ + compiledCommon: compiledCommon{ + path: mustCompilePath(t, "status", "phase"), + }, + LabelName: "phase", + List: []string{"foo", "bar"}, + }, wantResult: []eachValue{ + newEachValue(t, 0, "phase", "bar"), + newEachValue(t, 1, "phase", "foo"), + }}, + {name: "status_conditions", each: &compiledGauge{ + compiledCommon: compiledCommon{ + path: mustCompilePath(t, "status", "conditions", "[type=Ready]", "status"), + }, + }, wantResult: []eachValue{ + newEachValue(t, 1), + }}, + {name: "status_conditions_all", each: &compiledGauge{ + compiledCommon: compiledCommon{ + path: mustCompilePath(t, "status", "conditions"), + labelFromPath: map[string]valuePath{ + "type": mustCompilePath(t, "type"), + }, + }, + ValueFrom: mustCompilePath(t, "status"), + }, wantResult: []eachValue{ + newEachValue(t, 0, "type", "Provisioned"), + newEachValue(t, 1, "type", "Ready"), + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotResult, gotErrors := scrapeValuesFor(tt.each, cr) + assert.Equal(t, tt.wantResult, gotResult) + assert.Equal(t, tt.wantErrors, gotErrors) + }) + } +} + +func Test_compiledFamily_BaseLabels(t *testing.T) { + tests := []struct { + name string + fields compiledFamily + want map[string]string + }{ + {name: "both", fields: compiledFamily{ + Labels: map[string]string{ + "hello": "world", + }, + LabelFromPath: map[string]valuePath{ + "foo": mustCompilePath(t, "metadata", "annotations", "bar"), + }, + }, want: map[string]string{ + "hello": "world", + "foo": "baz", + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := tt.fields + if got := f.BaseLabels(cr); !reflect.DeepEqual(got, tt.want) { + t.Errorf("BaseLabels() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_eachValue_DefaultLabels(t *testing.T) { + tests := []struct { + name string + Labels map[string]string + defaults map[string]string + want map[string]string + }{ + {name: "all", Labels: map[string]string{ + "foo": "bar", + }, defaults: map[string]string{ + "foo": "baz", + "baz": "quxx", + }, want: map[string]string{ + "foo": "bar", + "baz": "quxx", + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := eachValue{ + Labels: tt.Labels, + } + e.DefaultLabels(tt.defaults) + assert.Equal(t, tt.want, e.Labels) + }) + } +} + +func Test_eachValue_ToMetric(t *testing.T) { + assert.Equal(t, &metric.Metric{ + Value: 123, + LabelKeys: []string{"bar", "foo", "quxx"}, + LabelValues: []string{"baz", "bar", "qux"}, + }, eachValue{ + Labels: map[string]string{ + "foo": "bar", + "bar": "baz", + "quxx": "qux", + }, + Value: 123, + }.ToMetric()) +} + +func Test_fullName(t *testing.T) { + type args struct { + resource Resource + f Generator + } + count := Generator{ + Name: "count", + } + tests := []struct { + name string + args args + want string + }{ + { + name: "defaults", + args: args{ + resource: r(nil), + f: count, + }, + want: "kube_customresource_count", + }, + { + name: "no prefix", + args: args{ + resource: r(pointer.String("")), + f: count, + }, + want: "count", + }, + { + name: "custom", + args: args{ + resource: r(pointer.String("bar_baz")), + f: count, + }, + want: "bar_baz_count", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := fullName(tt.args.resource, tt.args.f); got != tt.want { + t.Errorf("fullName() = %v, want %v", got, tt.want) + } + }) + } +} + +func r(metricNamePrefix *string) Resource { + return Resource{MetricNamePrefix: metricNamePrefix, GroupVersionKind: gkv("apps", "v1", "Deployment")} +} + +func gkv(group, version, kind string) GroupVersionKind { + return GroupVersionKind{ + Group: group, + Version: version, + Kind: kind, + } +} + +func Test_valuePath_Get(t *testing.T) { + + type testCase struct { + name string + p []string + want interface{} + } + tt := func(name string, want interface{}, path ...string) testCase { + return testCase{ + name: name, + p: path, + want: want, + } + } + tests := []testCase{ + tt("obj", float64(1), "spec", "replicas"), + tt("array", float64(66), "status", "condition_values", "[name=b]", "value"), + tt("array index", true, "spec", "order", "0", "value"), + tt("string", "bar", "metadata", "labels", "foo"), + tt("match number", false, "spec", "order", "[id=3]", "value"), + tt("match bool", float64(3), "spec", "order", "[value=false]", "id"), + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := mustCompilePath(t, tt.p...) + assert.Equal(t, tt.want, p.Get(cr)) + }) + } +} + +func newEachValue(t *testing.T, value float64, labels ...string) eachValue { + t.Helper() + if len(labels)%2 != 0 { + t.Fatalf("labels must be even: %v", labels) + } + m := make(map[string]string) + for i := 0; i < len(labels); i += 2 { + m[labels[i]] = labels[i+1] + } + return eachValue{ + Value: value, + Labels: m, + } +} + +func mustCompilePath(t *testing.T, path ...string) valuePath { + t.Helper() + out, err := compilePath(path) + if err != nil { + t.Fatalf("path %v: %v", path, err) + } + return out +} diff --git a/pkg/listwatch/listwatch.go b/pkg/listwatch/listwatch.go deleted file mode 100644 index 5f141fbd3f..0000000000 --- a/pkg/listwatch/listwatch.go +++ /dev/null @@ -1,265 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package listwatch - -import ( - "fmt" - "strings" - "sync" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/watch" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/tools/cache" -) - -// NewUnprivilegedNamespaceListWatchFromClient mimics -// cache.NewListWatchFromClient. -// It allows for the creation of a cache.ListWatch for namespaces from a client -// that does not have `List` privileges. If the slice of namespaces contains -// only v1.NamespaceAll, then this func assumes that the client has List and -// Watch privileges and returns a regular cache.ListWatch, since there is no -// other way to get all namespaces. -// -// The allowed namespaces and denied namespaces are mutually exclusive. -// See NewFilteredUnprivilegedNamespaceListWatchFromClient for a description on how they are applied. -func NewUnprivilegedNamespaceListWatchFromClient(c cache.Getter, allowedNamespaces, deniedNamespaces []string, fieldSelector fields.Selector) cache.ListerWatcher { - optionsModifier := func(options *metav1.ListOptions) { - options.FieldSelector = fieldSelector.String() - } - return NewFilteredUnprivilegedNamespaceListWatchFromClient(c, allowedNamespaces, deniedNamespaces, optionsModifier) -} - -// NewFilteredUnprivilegedNamespaceListWatchFromClient mimics -// cache.NewUnprivilegedNamespaceListWatchFromClient. -// It allows for the creation of a cache.ListWatch for allowed or denied namespaces -// from a client that does not have `List` privileges. -// -// If the given allowed namespaces contain only v1.NamespaceAll, -// then this function assumes that the client has List and -// Watch privileges and returns a regular cache.ListWatch, since there is no -// other way to get all namespaces. -// -// The given allowed and denied namespaces are mutually exclusive. -// If allowed namespaces contain multiple items, the given denied namespaces have no effect. -// If the allowed namespaces includes exactly one entry with the value v1.NamespaceAll (empty string), -// the given denied namespaces are applied. -func NewFilteredUnprivilegedNamespaceListWatchFromClient(c cache.Getter, allowedNamespaces, deniedNamespaces []string, optionsModifier func(options *metav1.ListOptions)) cache.ListerWatcher { - // If the only namespace given is `v1.NamespaceAll`, then this - // cache.ListWatch must be privileged. In this case, return a regular - // cache.ListWatch decorated with a denylist watcher - // filtering the given denied namespaces. - if IsAllNamespaces(allowedNamespaces) { - return newDenylistListerWatcher( - deniedNamespaces, - cache.NewFilteredListWatchFromClient(c, "namespaces", metav1.NamespaceAll, optionsModifier), - ) - } - listFunc := func(options metav1.ListOptions) (runtime.Object, error) { - optionsModifier(&options) - list := &v1.NamespaceList{} - for _, name := range allowedNamespaces { - result := &v1.Namespace{} - err := c.Get(). - Resource("namespaces"). - Name(name). - VersionedParams(&options, scheme.ParameterCodec). - Do(). - Into(result) - if err != nil { - return nil, err - } - list.Items = append(list.Items, *result) - } - return list, nil - } - watchFunc := func(_ metav1.ListOptions) (watch.Interface, error) { - // Since the client does not have Watch privileges, do not - // actually watch anything. Use a watch.FakeWatcher here to - // implement watch.Interface but not send any events. - return watch.NewFake(), nil - } - return &cache.ListWatch{ListFunc: listFunc, WatchFunc: watchFunc} -} - -// MultiNamespaceListerWatcher takes allowed and denied namespaces and a -// cache.ListerWatcher generator func and returns a single cache.ListerWatcher -// capable of operating on multiple namespaces. -// -// Allowed namespaces and denied namespaces are mutually exclusive. -// If allowed namespaces contain multiple items, the given denied namespaces have no effect. -// If the allowed namespaces includes exactly one entry with the value v1.NamespaceAll (empty string), -// the given denied namespaces are applied. -func MultiNamespaceListerWatcher(allowedNamespaces, deniedNamespaces []string, f func(string) cache.ListerWatcher) cache.ListerWatcher { - // If there is only one namespace then there is no need to create a - // multi lister watcher proxy. - if IsAllNamespaces(allowedNamespaces) { - return newDenylistListerWatcher(deniedNamespaces, f(allowedNamespaces[0])) - } - if len(allowedNamespaces) == 1 { - return f(allowedNamespaces[0]) - } - - var lws []cache.ListerWatcher - for _, n := range allowedNamespaces { - lws = append(lws, f(n)) - } - return multiListerWatcher(lws) -} - -// multiListerWatcher abstracts several cache.ListerWatchers, allowing them -// to be treated as a single cache.ListerWatcher. -type multiListerWatcher []cache.ListerWatcher - -// List implements the ListerWatcher interface. -// It combines the output of the List method of every ListerWatcher into -// a single result. -func (mlw multiListerWatcher) List(options metav1.ListOptions) (runtime.Object, error) { - l := metav1.List{} - var resourceVersions []string - for _, lw := range mlw { - list, err := lw.List(options) - if err != nil { - return nil, err - } - items, err := meta.ExtractList(list) - if err != nil { - return nil, err - } - metaObj, err := meta.ListAccessor(list) - if err != nil { - return nil, err - } - for _, item := range items { - l.Items = append(l.Items, runtime.RawExtension{Object: item.DeepCopyObject()}) - } - resourceVersions = append(resourceVersions, metaObj.GetResourceVersion()) - } - // Combine the resource versions so that the composite Watch method can - // distribute appropriate versions to each underlying Watch func. - l.ListMeta.ResourceVersion = strings.Join(resourceVersions, "/") - return &l, nil -} - -// Watch implements the ListerWatcher interface. -// It returns a watch.Interface that combines the output from the -// watch.Interface of every cache.ListerWatcher into a single result chan. -func (mlw multiListerWatcher) Watch(options metav1.ListOptions) (watch.Interface, error) { - resourceVersions := make([]string, len(mlw)) - // Allow resource versions to be "". - if options.ResourceVersion != "" { - rvs := strings.Split(options.ResourceVersion, "/") - if len(rvs) != len(mlw) { - return nil, fmt.Errorf("expected resource version to have %d parts to match the number of ListerWatchers", len(mlw)) - } - resourceVersions = rvs - } - return newMultiWatch(mlw, resourceVersions, options) -} - -// multiWatch abstracts multiple watch.Interface's, allowing them -// to be treated as a single watch.Interface. -type multiWatch struct { - result chan watch.Event - stopped chan struct{} - stoppers []func() -} - -// newMultiWatch returns a new multiWatch or an error if one of the underlying -// Watch funcs errored. The length of []cache.ListerWatcher and []string must -// match. -func newMultiWatch(lws []cache.ListerWatcher, resourceVersions []string, options metav1.ListOptions) (*multiWatch, error) { - var ( - result = make(chan watch.Event) - stopped = make(chan struct{}) - stoppers []func() - wg sync.WaitGroup - ) - - wg.Add(len(lws)) - - for i, lw := range lws { - o := options.DeepCopy() - o.ResourceVersion = resourceVersions[i] - w, err := lw.Watch(*o) - if err != nil { - return nil, err - } - - go func() { - defer wg.Done() - - for { - event, ok := <-w.ResultChan() - if !ok { - return - } - - select { - case result <- event: - case <-stopped: - return - } - } - }() - stoppers = append(stoppers, w.Stop) - } - - // result chan must be closed, - // once all event sender goroutines exited. - go func() { - wg.Wait() - close(result) - }() - - return &multiWatch{ - result: result, - stoppers: stoppers, - stopped: stopped, - }, nil -} - -// ResultChan implements the watch.Interface interface. -func (mw *multiWatch) ResultChan() <-chan watch.Event { - return mw.result -} - -// Stop implements the watch.Interface interface. -// It stops all of the underlying watch.Interfaces and closes the backing chan. -// Can safely be called more than once. -func (mw *multiWatch) Stop() { - select { - case <-mw.stopped: - // nothing to do, we are already stopped - default: - for _, stop := range mw.stoppers { - stop() - } - close(mw.stopped) - } - return -} - -// IsAllNamespaces checks if the given slice of namespaces -// contains only v1.NamespaceAll. -func IsAllNamespaces(namespaces []string) bool { - return len(namespaces) == 1 && namespaces[0] == v1.NamespaceAll -} diff --git a/pkg/listwatch/namespace_denylist.go b/pkg/listwatch/namespace_denylist.go deleted file mode 100644 index 3717f5a80b..0000000000 --- a/pkg/listwatch/namespace_denylist.go +++ /dev/null @@ -1,184 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package listwatch - -import ( - "fmt" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/watch" - "k8s.io/client-go/tools/cache" - "k8s.io/klog" -) - -// denylistListerWatcher implements cache.ListerWatcher -// which wraps a cache.ListerWatcher, -// filtering list results and watch events by denied namespaces. -type denylistListerWatcher struct { - denylist map[string]struct{} - next cache.ListerWatcher -} - -// newDenylistListerWatcher creates a cache.ListerWatcher -// wrapping the given next cache.ListerWatcher -// filtering lists and watch events by the given namespaces. -func newDenylistListerWatcher(namespaces []string, next cache.ListerWatcher) cache.ListerWatcher { - if len(namespaces) == 0 { - return next - } - - denylist := make(map[string]struct{}) - - for _, ns := range namespaces { - denylist[ns] = struct{}{} - } - - return &denylistListerWatcher{ - denylist: denylist, - next: next, - } -} - -// List lists the wrapped next listerwatcher List result, -// but filtering denied namespaces from the result. -func (w *denylistListerWatcher) List(options metav1.ListOptions) (runtime.Object, error) { - l := metav1.List{} - - list, err := w.next.List(options) - if err != nil { - klog.Errorf("error listing: %v", err) - return nil, err - } - - objs, err := meta.ExtractList(list) - if err != nil { - klog.Errorf("error extracting list: %v", err) - return nil, err - } - - metaObj, err := meta.ListAccessor(list) - if err != nil { - klog.Errorf("error getting list accessor: %v", err) - return nil, err - } - - for _, obj := range objs { - acc, err := meta.Accessor(obj) - if err != nil { - klog.Errorf("error getting meta accessor accessor for object %s: %v", fmt.Sprintf("%v", obj), err) - return nil, err - } - - if _, denied := w.denylist[getNamespace(acc)]; denied { - klog.V(8).Infof("denied %s", acc.GetSelfLink()) - continue - } - - klog.V(8).Infof("allowed %s", acc.GetSelfLink()) - - l.Items = append(l.Items, runtime.RawExtension{Object: obj.DeepCopyObject()}) - } - - l.ListMeta.ResourceVersion = metaObj.GetResourceVersion() - return &l, nil -} - -// Watch -func (w *denylistListerWatcher) Watch(options metav1.ListOptions) (watch.Interface, error) { - nextWatch, err := w.next.Watch(options) - if err != nil { - return nil, err - } - - return newDenylistWatch(w.denylist, nextWatch), nil -} - -// newDenylistWatch creates a new watch.Interface, -// wrapping the given next watcher, -// and filtering watch events by the given namespaces. -// -// It starts a new goroutine until either -// a) the result channel of the wrapped next watcher is closed, or -// b) Stop() was invoked on the returned watcher. -func newDenylistWatch(denylist map[string]struct{}, next watch.Interface) watch.Interface { - var ( - result = make(chan watch.Event) - proxy = watch.NewProxyWatcher(result) - ) - - go func() { - defer func() { - klog.V(8).Info("stopped denylist watcher") - // According to watch.Interface the result channel is supposed to be called - // in case of error or if the listwach is closed, see [1]. - // - // [1] https://github.com/kubernetes/apimachinery/blob/533d101be9a6450773bb2829bef282b6b7c4ff6d/pkg/watch/watch.go#L34-L37 - close(result) - }() - - for { - select { - case event, ok := <-next.ResultChan(): - if !ok { - klog.V(8).Info("result channel closed") - return - } - - acc, err := meta.Accessor(event.Object) - if err != nil { - // ignore this event, it doesn't implement the metav1.Object interface, - // hence we cannot determine its namespace. - klog.V(6).Infof("unexpected object type in event (%T): %v", event.Object, event.Object) - continue - } - - if _, denied := denylist[getNamespace(acc)]; denied { - klog.V(8).Infof("denied %s", acc.GetSelfLink()) - continue - } - - klog.V(8).Infof("allowed %s", acc.GetSelfLink()) - - select { - case result <- event: - klog.V(8).Infof("dispatched %s", acc.GetSelfLink()) - case <-proxy.StopChan(): - next.Stop() - return - } - case <-proxy.StopChan(): - next.Stop() - return - } - } - }() - - return proxy -} - -// getNamespace returns the namespace of the given object. -// If the object is itself a namespace, it returns the object's -// name. -func getNamespace(obj metav1.Object) string { - if _, ok := obj.(*v1.Namespace); ok { - return obj.GetName() - } - return obj.GetNamespace() -} diff --git a/pkg/metric/family.go b/pkg/metric/family.go index 98ac10037c..559f5645d5 100644 --- a/pkg/metric/family.go +++ b/pkg/metric/family.go @@ -20,12 +20,24 @@ import ( "strings" ) +// FamilyInterface interface for a family +type FamilyInterface interface { + Inspect(inspect func(Family)) + ByteSlice() []byte +} + // Family represents a set of metrics with the same name and help text. type Family struct { Name string + Type Type Metrics []*Metric } +// Inspect use to inspect the inside of a Family +func (f Family) Inspect(inspect func(Family)) { + inspect(f) +} + // ByteSlice returns the given Family in its string representation. func (f Family) ByteSlice() []byte { b := strings.Builder{} diff --git a/pkg/metric/generator.go b/pkg/metric/generator.go deleted file mode 100644 index 83439e91da..0000000000 --- a/pkg/metric/generator.go +++ /dev/null @@ -1,102 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package metric - -import ( - "strings" - - metricsstore "k8s.io/kube-state-metrics/pkg/metrics_store" -) - -// FamilyGenerator provides everything needed to generate a metric family with a -// Kubernetes object. -type FamilyGenerator struct { - Name string - Help string - Type Type - GenerateFunc func(obj interface{}) *Family -} - -// Generate calls the FamilyGenerator.GenerateFunc and gives the family its -// name. The reasoning behind injecting the name at such a late point in time is -// deduplication in the code, preventing typos made by developers as -// well as saving memory. -func (g *FamilyGenerator) Generate(obj interface{}) *Family { - family := g.GenerateFunc(obj) - family.Name = g.Name - return family -} - -func (g *FamilyGenerator) generateHeader() string { - header := strings.Builder{} - header.WriteString("# HELP ") - header.WriteString(g.Name) - header.WriteByte(' ') - header.WriteString(g.Help) - header.WriteByte('\n') - header.WriteString("# TYPE ") - header.WriteString(g.Name) - header.WriteByte(' ') - header.WriteString(string(g.Type)) - - return header.String() -} - -// ExtractMetricFamilyHeaders takes in a slice of FamilyGenerator metrics and -// returns the extracted headers. -func ExtractMetricFamilyHeaders(families []FamilyGenerator) []string { - headers := make([]string, len(families)) - - for i, f := range families { - headers[i] = f.generateHeader() - } - - return headers -} - -// ComposeMetricGenFuncs takes a slice of metric families and returns a function -// that composes their metric generation functions into a single one. -func ComposeMetricGenFuncs(familyGens []FamilyGenerator) func(obj interface{}) []metricsstore.FamilyByteSlicer { - return func(obj interface{}) []metricsstore.FamilyByteSlicer { - families := make([]metricsstore.FamilyByteSlicer, len(familyGens)) - - for i, gen := range familyGens { - families[i] = gen.Generate(obj) - } - - return families - } -} - -type whiteBlackLister interface { - IsIncluded(string) bool - IsExcluded(string) bool -} - -// FilterMetricFamilies takes a white- and a blacklist and a slice of metric -// families and returns a filtered slice. -func FilterMetricFamilies(l whiteBlackLister, families []FamilyGenerator) []FamilyGenerator { - filtered := []FamilyGenerator{} - - for _, f := range families { - if l.IsIncluded(f.Name) { - filtered = append(filtered, f) - } - } - - return filtered -} diff --git a/pkg/metric/metric.go b/pkg/metric/metric.go index 96bf8c8063..007eb9fa87 100644 --- a/pkg/metric/metric.go +++ b/pkg/metric/metric.go @@ -38,13 +38,19 @@ var ( ) // Type represents the type of a metric e.g. a counter. See -// https://prometheus.io/docs/concepts/metric_types/. +// https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#metric-types. type Type string -// Gauge defines a Prometheus gauge. +// Gauge defines a OpenMetrics gauge. var Gauge Type = "gauge" -// Counter defines a Prometheus counter. +// Info defines an OpenMetrics info. +var Info Type = "info" + +// StateSet defines an OpenMetrics stateset. +var StateSet Type = "stateset" + +// Counter defines a OpenMetrics counter. var Counter Type = "counter" // Metric represents a single time series. diff --git a/pkg/metric_generator/filter.go b/pkg/metric_generator/filter.go new file mode 100644 index 0000000000..10a5636ac9 --- /dev/null +++ b/pkg/metric_generator/filter.go @@ -0,0 +1,61 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generator + +// FamilyGeneratorFilter represents a filter which decides whether a metric +// family is exposed by the store or not +type FamilyGeneratorFilter interface { + + // Test returns true if it passes the criteria of the filter, otherwise it + // will return false and the metric family is not exposed by the store + Test(generator FamilyGenerator) bool +} + +// CompositeFamilyGeneratorFilter is composite for combining multiple filters +type CompositeFamilyGeneratorFilter struct { + filters []FamilyGeneratorFilter +} + +// Test tests the generator by passing it through the filters contained within the composite +// and return false if the generator does not match all the filters +func (composite CompositeFamilyGeneratorFilter) Test(generator FamilyGenerator) bool { + for _, filter := range composite.filters { + if !filter.Test(generator) { + return false + } + } + return true +} + +// FilterFamilyGenerators filters a given slice of family generators based upon a given filter +// and returns a slice containing the family generators which passed the filter criteria +func FilterFamilyGenerators(filter FamilyGeneratorFilter, families []FamilyGenerator) []FamilyGenerator { + var filtered []FamilyGenerator + + for _, family := range families { + if filter.Test(family) { + filtered = append(filtered, family) + } + } + + return filtered +} + +// NewCompositeFamilyGeneratorFilter combines multiple family generators filters into one composite filter +func NewCompositeFamilyGeneratorFilter(filters ...FamilyGeneratorFilter) CompositeFamilyGeneratorFilter { + return CompositeFamilyGeneratorFilter{filters} +} diff --git a/pkg/metric_generator/generator.go b/pkg/metric_generator/generator.go new file mode 100644 index 0000000000..e1cce4906d --- /dev/null +++ b/pkg/metric_generator/generator.go @@ -0,0 +1,124 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generator + +import ( + "fmt" + "strings" + + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" +) + +// FamilyGenerator provides everything needed to generate a metric family with a +// Kubernetes object. +// DeprecatedVersion is defined only if the metric for which this options applies is, +// in fact, deprecated. +type FamilyGenerator struct { + Name string + Help string + Type metric.Type + OptIn bool + DeprecatedVersion string + StabilityLevel basemetrics.StabilityLevel + GenerateFunc func(obj interface{}) *metric.Family +} + +// NewFamilyGeneratorWithStability creates new FamilyGenerator instances with metric +// stabilityLevel. +func NewFamilyGeneratorWithStability(name string, help string, metricType metric.Type, stabilityLevel basemetrics.StabilityLevel, deprecatedVersion string, generateFunc func(obj interface{}) *metric.Family) *FamilyGenerator { + f := &FamilyGenerator{ + Name: name, + Type: metricType, + Help: help, + OptIn: false, + StabilityLevel: stabilityLevel, + DeprecatedVersion: deprecatedVersion, + GenerateFunc: generateFunc, + } + if deprecatedVersion != "" { + f.Help = fmt.Sprintf("(Deprecated since %s) %s", deprecatedVersion, help) + } + return f +} + +// NewOptInFamilyGenerator creates new FamilyGenerator instances for opt-in metric families. +func NewOptInFamilyGenerator(name string, help string, metricType metric.Type, stabilityLevel basemetrics.StabilityLevel, deprecatedVersion string, generateFunc func(obj interface{}) *metric.Family) *FamilyGenerator { + f := NewFamilyGeneratorWithStability(name, help, metricType, stabilityLevel, + deprecatedVersion, generateFunc) + f.OptIn = true + return f +} + +// Generate calls the FamilyGenerator.GenerateFunc and gives the family its +// name. The reasoning behind injecting the name at such a late point in time is +// deduplication in the code, preventing typos made by developers as +// well as saving memory. +func (g *FamilyGenerator) Generate(obj interface{}) *metric.Family { + family := g.GenerateFunc(obj) + family.Name = g.Name + family.Type = g.Type + return family +} + +func (g *FamilyGenerator) generateHeader() string { + header := strings.Builder{} + header.WriteString("# HELP ") + header.WriteString(g.Name) + header.WriteByte(' ') + // TODO(#1833): remove if-else after all metrics are attached with right + // StabilityLevel. + if g.StabilityLevel == basemetrics.STABLE { + header.WriteString(fmt.Sprintf("[%v] %v", g.StabilityLevel, g.Help)) + } else { + header.WriteString(g.Help) + } + header.WriteByte('\n') + header.WriteString("# TYPE ") + header.WriteString(g.Name) + header.WriteByte(' ') + header.WriteString(string(g.Type)) + + return header.String() +} + +// ExtractMetricFamilyHeaders takes in a slice of FamilyGenerator metrics and +// returns the extracted headers. +func ExtractMetricFamilyHeaders(families []FamilyGenerator) []string { + headers := make([]string, len(families)) + + for i, f := range families { + headers[i] = f.generateHeader() + } + + return headers +} + +// ComposeMetricGenFuncs takes a slice of metric families and returns a function +// that composes their metric generation functions into a single one. +func ComposeMetricGenFuncs(familyGens []FamilyGenerator) func(obj interface{}) []metric.FamilyInterface { + return func(obj interface{}) []metric.FamilyInterface { + families := make([]metric.FamilyInterface, len(familyGens)) + + for i, gen := range familyGens { + families[i] = gen.Generate(obj) + } + + return families + } +} diff --git a/pkg/metrics_store/metrics_store.go b/pkg/metrics_store/metrics_store.go index 116ab303fb..80d7149781 100644 --- a/pkg/metrics_store/metrics_store.go +++ b/pkg/metrics_store/metrics_store.go @@ -17,18 +17,13 @@ limitations under the License. package metricsstore import ( - "io" "sync" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/types" -) -// FamilyByteSlicer represents a metric family that can be converted to its string -// representation. -type FamilyByteSlicer interface { - ByteSlice() []byte -} + "k8s.io/kube-state-metrics/v2/pkg/metric" +) // MetricsStore implements the k8s.io/client-go/tools/cache.Store // interface. Instead of storing entire Kubernetes objects, it stores metrics @@ -48,11 +43,11 @@ type MetricsStore struct { // generateMetricsFunc generates metrics based on a given Kubernetes object // and returns them grouped by metric family. - generateMetricsFunc func(interface{}) []FamilyByteSlicer + generateMetricsFunc func(interface{}) []metric.FamilyInterface } // NewMetricsStore returns a new MetricsStore -func NewMetricsStore(headers []string, generateFunc func(interface{}) []FamilyByteSlicer) *MetricsStore { +func NewMetricsStore(headers []string, generateFunc func(interface{}) []metric.FamilyInterface) *MetricsStore { return &MetricsStore{ generateMetricsFunc: generateFunc, headers: headers, @@ -148,18 +143,3 @@ func (s *MetricsStore) Replace(list []interface{}, _ string) error { func (s *MetricsStore) Resync() error { return nil } - -// WriteAll writes all metrics of the store into the given writer, zipped with the -// help text of each metric family. -func (s *MetricsStore) WriteAll(w io.Writer) { - s.mutex.RLock() - defer s.mutex.RUnlock() - - for i, help := range s.headers { - w.Write([]byte(help)) - w.Write([]byte{'\n'}) - for _, metricFamilies := range s.metrics { - w.Write(metricFamilies[i]) - } - } -} diff --git a/pkg/metrics_store/metrics_store_test.go b/pkg/metrics_store/metrics_store_test.go index b8c6ec683f..b32e1bcb38 100644 --- a/pkg/metrics_store/metrics_store_test.go +++ b/pkg/metrics_store/metrics_store_test.go @@ -25,6 +25,8 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + + "k8s.io/kube-state-metrics/v2/pkg/metric" ) // Mock metricFamily instead of importing /pkg/metric to prevent cyclic @@ -41,17 +43,24 @@ func (f *metricFamily) ByteSlice() []byte { func TestObjectsSameNameDifferentNamespaces(t *testing.T) { serviceIDS := []string{"a", "b"} - genFunc := func(obj interface{}) []FamilyByteSlicer { + genFunc := func(obj interface{}) []metric.FamilyInterface { o, err := meta.Accessor(obj) if err != nil { t.Fatal(err) } - metricFamily := metricFamily{ - []byte(fmt.Sprintf("kube_service_info{uid=\"%v\"} 1", string(o.GetUID()))), + metricFamily := metric.Family{ + Name: "kube_service_info", + Metrics: []*metric.Metric{ + { + LabelKeys: []string{"uid"}, + LabelValues: []string{string(o.GetUID())}, + Value: float64(1), + }, + }, } - return []FamilyByteSlicer{&metricFamily} + return []metric.FamilyInterface{&metricFamily} } ms := NewMetricsStore([]string{"Information about service."}, genFunc) @@ -72,7 +81,11 @@ func TestObjectsSameNameDifferentNamespaces(t *testing.T) { } w := strings.Builder{} - ms.WriteAll(&w) + mw := NewMetricsWriter(ms) + err := mw.WriteAll(&w) + if err != nil { + t.Fatalf("failed to write metrics: %v", err) + } m := w.String() for _, id := range serviceIDS { diff --git a/pkg/metrics_store/metrics_writer.go b/pkg/metrics_store/metrics_writer.go new file mode 100644 index 0000000000..7cc92c298b --- /dev/null +++ b/pkg/metrics_store/metrics_writer.go @@ -0,0 +1,77 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metricsstore + +import ( + "fmt" + "io" +) + +// MetricsWriterList represent a list of MetricsWriter +type MetricsWriterList []*MetricsWriter + +// MetricsWriter is a struct that holds multiple MetricsStore(s) and +// implements the MetricsWriter interface. +// It should be used with stores which have the same metric headers. +// +// MetricsWriter writes out metrics from the underlying stores so that +// metrics with the same name coming from different stores end up grouped together. +// It also ensures that the metric headers are only written out once. +type MetricsWriter struct { + stores []*MetricsStore +} + +// NewMetricsWriter creates a new MetricsWriter. +func NewMetricsWriter(stores ...*MetricsStore) *MetricsWriter { + return &MetricsWriter{ + stores: stores, + } +} + +// WriteAll writes out metrics from the underlying stores to the given writer. +// +// WriteAll writes metrics so that the ones with the same name +// are grouped together when written out. +func (m MetricsWriter) WriteAll(w io.Writer) error { + if len(m.stores) == 0 { + return nil + } + + for _, s := range m.stores { + s.mutex.RLock() + defer func(s *MetricsStore) { + s.mutex.RUnlock() + }(s) + } + + for i, help := range m.stores[0].headers { + _, err := w.Write([]byte(help + "\n")) + if err != nil { + return fmt.Errorf("failed to write help text: %v", err) + } + + for _, s := range m.stores { + for _, metricFamilies := range s.metrics { + _, err := w.Write(metricFamilies[i]) + if err != nil { + return fmt.Errorf("failed to write metrics family: %v", err) + } + } + } + } + return nil +} diff --git a/pkg/metrics_store/metrics_writer_test.go b/pkg/metrics_store/metrics_writer_test.go new file mode 100644 index 0000000000..b13c160555 --- /dev/null +++ b/pkg/metrics_store/metrics_writer_test.go @@ -0,0 +1,233 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metricsstore_test + +import ( + "strings" + "testing" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + metricsstore "k8s.io/kube-state-metrics/v2/pkg/metrics_store" +) + +func TestWriteAllWithSingleStore(t *testing.T) { + genFunc := func(obj interface{}) []metric.FamilyInterface { + o, err := meta.Accessor(obj) + if err != nil { + t.Fatal(err) + } + + mf1 := metric.Family{ + Name: "kube_service_info_1", + Metrics: []*metric.Metric{ + { + LabelKeys: []string{"namespace", "uid"}, + LabelValues: []string{o.GetNamespace(), string(o.GetUID())}, + Value: float64(1), + }, + }, + } + + mf2 := metric.Family{ + Name: "kube_service_info_2", + Metrics: []*metric.Metric{ + { + LabelKeys: []string{"namespace", "uid"}, + LabelValues: []string{o.GetNamespace(), string(o.GetUID())}, + Value: float64(1), + }, + }, + } + + return []metric.FamilyInterface{&mf1, &mf2} + } + store := metricsstore.NewMetricsStore([]string{"Info 1 about services", "Info 2 about services"}, genFunc) + svcs := []v1.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + UID: "a1", + Name: "service", + Namespace: "a", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + UID: "a2", + Name: "service", + Namespace: "a", + }, + }, + } + for _, s := range svcs { + svc := s + if err := store.Add(&svc); err != nil { + t.Fatal(err) + } + } + + multiNsWriter := metricsstore.NewMetricsWriter(store) + w := strings.Builder{} + err := multiNsWriter.WriteAll(&w) + if err != nil { + t.Fatalf("failed to write metrics: %v", err) + } + result := w.String() + + resultLines := strings.Split(strings.TrimRight(result, "\n"), "\n") + if len(resultLines) != 6 { + t.Fatalf("Invalid number of series, got %d, want %d", len(resultLines), 6) + } + if resultLines[0] != "Info 1 about services" { + t.Fatalf("Invalid metrics header on line 0, got %s, want %s", resultLines[0], "Info 1 about services") + } + if resultLines[3] != "Info 2 about services" { + t.Fatalf("Invalid metrics header on line 3, got %s, want %s", resultLines[3], "Info 2 about services") + } + + expectedSeries := []string{ + `kube_service_info_1{namespace="a",uid="a1"} 1`, + `kube_service_info_1{namespace="a",uid="a2"} 1`, + `kube_service_info_2{namespace="a",uid="a1"} 1`, + `kube_service_info_2{namespace="a",uid="a2"} 1`, + } + + for _, series := range expectedSeries { + if !strings.Contains(result, series) { + t.Fatalf("Did not find expected series %s", series) + } + } +} + +func TestWriteAllWithMultipleStores(t *testing.T) { + genFunc := func(obj interface{}) []metric.FamilyInterface { + o, err := meta.Accessor(obj) + if err != nil { + t.Fatal(err) + } + + mf1 := metric.Family{ + Name: "kube_service_info_1", + Metrics: []*metric.Metric{ + { + LabelKeys: []string{"namespace", "uid"}, + LabelValues: []string{o.GetNamespace(), string(o.GetUID())}, + Value: float64(1), + }, + }, + } + + mf2 := metric.Family{ + Name: "kube_service_info_2", + Metrics: []*metric.Metric{ + { + LabelKeys: []string{"namespace", "uid"}, + LabelValues: []string{o.GetNamespace(), string(o.GetUID())}, + Value: float64(1), + }, + }, + } + + return []metric.FamilyInterface{&mf1, &mf2} + } + s1 := metricsstore.NewMetricsStore([]string{"Info 1 about services", "Info 2 about services"}, genFunc) + svcs1 := []v1.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + UID: "a1", + Name: "service", + Namespace: "a", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + UID: "a2", + Name: "service", + Namespace: "a", + }, + }, + } + for _, s := range svcs1 { + svc := s + if err := s1.Add(&svc); err != nil { + t.Fatal(err) + } + } + + svcs2 := []v1.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + UID: "b1", + Name: "service", + Namespace: "b", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + UID: "b2", + Name: "service", + Namespace: "b", + }, + }, + } + s2 := metricsstore.NewMetricsStore([]string{"Info 1 about services", "Info 2 about services"}, genFunc) + for _, s := range svcs2 { + svc := s + if err := s2.Add(&svc); err != nil { + t.Fatal(err) + } + } + + multiNsWriter := metricsstore.NewMetricsWriter(s1, s2) + w := strings.Builder{} + err := multiNsWriter.WriteAll(&w) + if err != nil { + t.Fatalf("failed to write metrics: %v", err) + } + result := w.String() + + resultLines := strings.Split(strings.TrimRight(result, "\n"), "\n") + if len(resultLines) != 10 { + t.Fatalf("Invalid number of series, got %d, want %d", len(resultLines), 10) + } + if resultLines[0] != "Info 1 about services" { + t.Fatalf("Invalid metrics header on line 0, got %s, want %s", resultLines[0], "Info 1 about services") + } + if resultLines[5] != "Info 2 about services" { + t.Fatalf("Invalid metrics header on line 0, got %s, want %s", resultLines[5], "Info 2 about services") + } + + expectedSeries := []string{ + `kube_service_info_1{namespace="a",uid="a1"} 1`, + `kube_service_info_1{namespace="a",uid="a2"} 1`, + `kube_service_info_1{namespace="b",uid="b1"} 1`, + `kube_service_info_1{namespace="b",uid="b2"} 1`, + `kube_service_info_2{namespace="a",uid="a1"} 1`, + `kube_service_info_2{namespace="a",uid="a2"} 1`, + `kube_service_info_2{namespace="b",uid="b1"} 1`, + `kube_service_info_2{namespace="b",uid="b2"} 1`, + } + + for _, series := range expectedSeries { + if !strings.Contains(result, series) { + t.Fatalf("Did not find expected series %s", series) + } + } +} diff --git a/pkg/metricshandler/metrics_handler.go b/pkg/metricshandler/metrics_handler.go index 47f8b4d89b..a215fee6a3 100644 --- a/pkg/metricshandler/metrics_handler.go +++ b/pkg/metricshandler/metrics_handler.go @@ -19,23 +19,26 @@ package metricshandler import ( "compress/gzip" "context" + "errors" + "fmt" "io" "net/http" "strconv" "strings" "sync" - "github.com/pkg/errors" + "github.com/prometheus/common/expfmt" + appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" - "k8s.io/klog" + "k8s.io/klog/v2" - "k8s.io/kube-state-metrics/internal/store" - metricsstore "k8s.io/kube-state-metrics/pkg/metrics_store" - "k8s.io/kube-state-metrics/pkg/options" + ksmtypes "k8s.io/kube-state-metrics/v2/pkg/builder/types" + metricsstore "k8s.io/kube-state-metrics/v2/pkg/metrics_store" + "k8s.io/kube-state-metrics/v2/pkg/options" ) // MetricsHandler is a http.Handler that exposes the main kube-state-metrics @@ -43,20 +46,20 @@ import ( type MetricsHandler struct { opts *options.Options kubeClient kubernetes.Interface - storeBuilder *store.Builder + storeBuilder ksmtypes.BuilderInterface enableGZIPEncoding bool cancel func() - // mtx protects stores, curShard, and curTotalShards + // mtx protects metricsWriters, curShard, and curTotalShards mtx *sync.RWMutex - stores []*metricsstore.MetricsStore + metricsWriters metricsstore.MetricsWriterList curShard int32 curTotalShards int } // New creates and returns a new MetricsHandler with the given options. -func New(opts *options.Options, kubeClient kubernetes.Interface, storeBuilder *store.Builder, enableGZIPEncoding bool) *MetricsHandler { +func New(opts *options.Options, kubeClient kubernetes.Interface, storeBuilder ksmtypes.BuilderInterface, enableGZIPEncoding bool) *MetricsHandler { return &MetricsHandler{ opts: opts, kubeClient: kubeClient, @@ -76,12 +79,12 @@ func (m *MetricsHandler) ConfigureSharding(ctx context.Context, shard int32, tot m.cancel() } if totalShards != 1 { - klog.Infof("configuring sharding of this instance to be shard index %d (zero-indexed) out of %d total shards", shard, totalShards) + klog.InfoS("Configuring sharding of this instance to be shard index (zero-indexed) out of total shards", "shard", shard, "totalShards", totalShards) } ctx, m.cancel = context.WithCancel(ctx) m.storeBuilder.WithSharding(shard, totalShards) m.storeBuilder.WithContext(ctx) - m.stores = m.storeBuilder.Build() + m.metricsWriters = m.storeBuilder.Build() m.curShard = shard m.curTotalShards = totalShards } @@ -93,17 +96,17 @@ func (m *MetricsHandler) Run(ctx context.Context) error { autoSharding := len(m.opts.Pod) > 0 && len(m.opts.Namespace) > 0 if !autoSharding { - klog.Info("Autosharding disabled") + klog.InfoS("Autosharding disabled") m.ConfigureSharding(ctx, m.opts.Shard, m.opts.TotalShards) <-ctx.Done() return ctx.Err() } - klog.Infof("Autosharding enabled with pod=%v pod_namespace=%v", m.opts.Pod, m.opts.Namespace) - klog.Infof("Auto detecting sharding settings.") + klog.InfoS("Autosharding enabled with pod", "pod", klog.KRef(m.opts.Namespace, m.opts.Pod)) + klog.InfoS("Auto detecting sharding settings") ss, err := detectStatefulSet(m.kubeClient, m.opts.Pod, m.opts.Namespace) if err != nil { - return errors.Wrap(err, "detect StatefulSet") + return fmt.Errorf("detect StatefulSet: %w", err) } statefulSetName := ss.Name @@ -124,7 +127,7 @@ func (m *MetricsHandler) Run(ctx context.Context) error { shard, totalShards, err := shardingSettingsFromStatefulSet(ss, m.opts.Pod) if err != nil { - klog.Errorf("detect sharding settings from StatefulSet: %v", err) + klog.ErrorS(err, "Detected sharding settings from StatefulSet") return } @@ -151,7 +154,7 @@ func (m *MetricsHandler) Run(ctx context.Context) error { shard, totalShards, err := shardingSettingsFromStatefulSet(cur, m.opts.Pod) if err != nil { - klog.Errorf("detect sharding settings from StatefulSet: %v", err) + klog.ErrorS(err, "Detected sharding settings from StatefulSet") return } @@ -174,15 +177,21 @@ func (m *MetricsHandler) Run(ctx context.Context) error { return ctx.Err() } -// ServeHTTP implements the http.Handler interface. It writes the metrics in -// its stores to the response body. +// ServeHTTP implements the http.Handler interface. It writes all generated +// metrics to the response body. func (m *MetricsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { m.mtx.RLock() defer m.mtx.RUnlock() resHeader := w.Header() var writer io.Writer = w - resHeader.Set("Content-Type", `text/plain; version=`+"0.0.4") + contentType := expfmt.NegotiateIncludingOpenMetrics(r.Header) + + // We do not support protobuf at the moment. Fall back to FmtText if the negotiated exposition format is not FmtOpenMetrics See: https://github.com/kubernetes/kube-state-metrics/issues/2022 + if contentType != expfmt.FmtOpenMetrics { + contentType = expfmt.FmtText + } + resHeader.Set("Content-Type", string(contentType)) if m.enableGZIPEncoding { // Gzip response if requested. Taken from @@ -198,20 +207,31 @@ func (m *MetricsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } - for _, s := range m.stores { - s.WriteAll(w) + for _, w := range m.metricsWriters { + err := w.WriteAll(writer) + if err != nil { + klog.ErrorS(err, "Failed to write metrics") + } + } + + // If we send openmetrics, we need to include a EOF directive + if contentType == expfmt.FmtOpenMetrics { + w.Write([]byte("# EOF\n")) } // In case we gzipped the response, we have to close the writer. if closer, ok := writer.(io.Closer); ok { - closer.Close() + err := closer.Close() + if err != nil { + klog.ErrorS(err, "Failed to close the writer") + } } } func shardingSettingsFromStatefulSet(ss *appsv1.StatefulSet, podName string) (nominal int32, totalReplicas int, err error) { nominal, err = detectNominalFromPod(ss.Name, podName) if err != nil { - return 0, 0, errors.Wrap(err, "detecting Pod nominal") + return 0, 0, fmt.Errorf("detecting Pod nominal: %w", err) } totalReplicas = 1 @@ -225,18 +245,18 @@ func shardingSettingsFromStatefulSet(ss *appsv1.StatefulSet, podName string) (no func detectNominalFromPod(statefulSetName, podName string) (int32, error) { nominalString := strings.TrimPrefix(podName, statefulSetName+"-") - nominal, err := strconv.Atoi(nominalString) + nominal, err := strconv.ParseInt(nominalString, 10, 32) if err != nil { - return 0, errors.Wrapf(err, "failed to detect shard index for Pod %s of StatefulSet %s, parsed %s", podName, statefulSetName, nominalString) + return 0, fmt.Errorf("failed to detect shard index for Pod %s of StatefulSet %s, parsed %s: %w", podName, statefulSetName, nominalString, err) } return int32(nominal), nil } func detectStatefulSet(kubeClient kubernetes.Interface, podName, namespaceName string) (*appsv1.StatefulSet, error) { - p, err := kubeClient.CoreV1().Pods(namespaceName).Get(podName, metav1.GetOptions{}) + p, err := kubeClient.CoreV1().Pods(namespaceName).Get(context.TODO(), podName, metav1.GetOptions{}) if err != nil { - return nil, errors.Wrapf(err, "retrieve pod %s for sharding", podName) + return nil, fmt.Errorf("retrieve pod %s for sharding: %w", podName, err) } owners := p.GetOwnerReferences() @@ -245,13 +265,13 @@ func detectStatefulSet(kubeClient kubernetes.Interface, podName, namespaceName s continue } - ss, err := kubeClient.AppsV1().StatefulSets(namespaceName).Get(o.Name, metav1.GetOptions{}) + ss, err := kubeClient.AppsV1().StatefulSets(namespaceName).Get(context.TODO(), o.Name, metav1.GetOptions{}) if err != nil { - return nil, errors.Wrapf(err, "retrieve shard's StatefulSet: %s/%s", namespaceName, o.Name) + return nil, fmt.Errorf("retrieve shard's StatefulSet: %s/%s: %w", namespaceName, o.Name, err) } return ss, nil } - return nil, errors.Errorf("no suitable statefulset found for auto detecting sharding for Pod %s/%s", namespaceName, podName) + return nil, fmt.Errorf("no suitable statefulset found for auto detecting sharding for Pod %s/%s", namespaceName, podName) } diff --git a/pkg/optin/optin.go b/pkg/optin/optin.go new file mode 100644 index 0000000000..7f57cbfbdb --- /dev/null +++ b/pkg/optin/optin.go @@ -0,0 +1,72 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package optin + +import ( + "regexp" + "sort" + "strings" + + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +// MetricFamilyFilter filters metric families which are defined as opt-in by their generator.FamilyGenerator +type MetricFamilyFilter struct { + metrics []*regexp.Regexp +} + +// Test tests if a given generator is an opt-in metric family and was passed as an opt-in metric family at startup +func (filter MetricFamilyFilter) Test(generator generator.FamilyGenerator) bool { + if !generator.OptIn { + return true + } + for _, metric := range filter.metrics { + if metric.MatchString(generator.Name) { + return true + } + } + return false +} + +// Status returns the metrics contained within the filter as a comma-separated string +func (filter MetricFamilyFilter) Status() string { + asStrings := make([]string, 0) + for _, metric := range filter.metrics { + asStrings = append(asStrings, metric.String()) + } + // sort the strings for the sake of ux such that the resulting status is consistent + sort.Strings(asStrings) + return strings.Join(asStrings, ", ") +} + +// Count returns the amount of metrics contained within the filter +func (filter MetricFamilyFilter) Count() int { + return len(filter.metrics) +} + +// NewMetricFamilyFilter creates new MetricFamilyFilter instances. +func NewMetricFamilyFilter(metrics map[string]struct{}) (*MetricFamilyFilter, error) { + regexes := make([]*regexp.Regexp, 0) + for metric := range metrics { + regex, err := regexp.Compile(metric) + if err != nil { + return nil, err + } + regexes = append(regexes, regex) + } + return &MetricFamilyFilter{regexes}, nil +} diff --git a/pkg/optin/optin_test.go b/pkg/optin/optin_test.go new file mode 100644 index 0000000000..3a36f899a3 --- /dev/null +++ b/pkg/optin/optin_test.go @@ -0,0 +1,111 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package optin + +import ( + "testing" + + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +func TestFilter(t *testing.T) { + tests := []struct { + MetricFamily string + IsOptIn bool + OptInMetric string + Want bool + }{ + {"kube_pod_container_status_running", true, "kube_pod_container_status_.+", true}, + {"kube_pod_container_status_terminated", true, "kube_pod_container_status_running", false}, + {"kube_pod_container_status_reason", true, "kube_pod_container_status_(running|terminated)", false}, + {"kube_node_info", false, "", true}, + } + + for _, test := range tests { + filter, err := NewMetricFamilyFilter(map[string]struct{}{ + test.OptInMetric: {}, + }) + if err != nil { + t.Errorf("did not expect NewMetricFamilyFilter to fail, the error is %v", err) + } + + optInFamilyGenerator := *generator.NewFamilyGeneratorWithStability( + test.MetricFamily, + "", + metric.Gauge, + basemetrics.ALPHA, + "", + func(_ interface{}) *metric.Family { + return nil + }, + ) + optInFamilyGenerator.OptIn = test.IsOptIn + + result := filter.Test(optInFamilyGenerator) + if result != test.Want { + t.Errorf("the metric family did not pass the filter, got: %v, want: %v", result, test.Want) + } + } +} + +func TestRegexParsing(t *testing.T) { + t.Run("should fail if an invalid regular expression is passed in", func(t *testing.T) { + _, err := NewMetricFamilyFilter(map[string]struct{}{"*_pod_info": {}}) + if err == nil { + t.Errorf("expected NewMetricFamilyFilter to fail for invalid regex pattern") + } + }) + + t.Run("should succeed when valid regular expressions are passed in", func(t *testing.T) { + _, err := NewMetricFamilyFilter(map[string]struct{}{"kube_.*_info": {}}) + if err != nil { + t.Errorf("expected NewMetricFamilyFilter to succeed, but failed : %v", err) + } + }) +} + +func TestStatus(t *testing.T) { + filter, err := NewMetricFamilyFilter(map[string]struct{}{ + "kube_pod_container_status_running": {}, + "kube_pod_container_status_terminated": {}, + }) + if err != nil { + t.Errorf("did not expect NewMetricFamilyFilter to fail, the error is %v", err) + } + + status := filter.Status() + if status != "kube_pod_container_status_running, kube_pod_container_status_terminated" { + t.Errorf("the metric family filter did not return the correct status, got: \"%v\", want: \"%v\"", status, "kube_pod_container_status_running, kube_pod_container_status_terminated") + } +} + +func TestCount(t *testing.T) { + filter, err := NewMetricFamilyFilter(map[string]struct{}{ + "kube_pod_container_status_running": {}, + "kube_pod_container_status_terminated": {}, + }) + if err != nil { + t.Errorf("did not expect NewMetricFamilyFilter to fail, the error is %v", err) + } + + if filter.Count() != 2 { + t.Errorf("the metric family filter did not return the correct amount of filters in it, got: %v, want: %v", filter.Count(), 2) + } +} diff --git a/pkg/options/autoload.go b/pkg/options/autoload.go new file mode 100644 index 0000000000..5d51a3d3af --- /dev/null +++ b/pkg/options/autoload.go @@ -0,0 +1,143 @@ +/* +Copyright 2022 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "os" + + "github.com/spf13/cobra" + "k8s.io/klog/v2" +) + +const autoloadZsh = `Generate the autocompletion script for the zsh shell. + +If shell completion is not already enabled in your environment you will need +to enable it. You can execute the following once: + + echo "autoload -U compinit; compinit" >> ~/.zshrc + +To load completions in your current shell session: + + source <(kube-state-metrics completion zsh); compdef _kube-state-metrics kube-state-metrics + +To load completions for every new session, execute once: + +#### Linux: + + kube-state-metrics completion zsh > "${fpath[1]}/_kube-state-metrics" + +#### macOS: + + kube-state-metrics completion zsh > $(brew --prefix)/share/zsh/site-functions/_kube-state-metrics + +You will need to start a new shell for this setup to take effect. + +Usage: + kube-state-metrics completion zsh [flags] + +Flags: + --no-descriptions disable completion descriptions +` +const autoloadBash = `Generate the autocompletion script for the bash shell. + +This script depends on the 'bash-completion' package. +If it is not installed already, you can install it via your OS's package manager. + +To load completions in your current shell session: + + source <(kube-state-metrics completion bash) + +To load completions for every new session, execute once: + +#### Linux: + + kube-state-metrics completion bash > /etc/bash_completion.d/kube-state-metrics + +#### macOS: + + kube-state-metrics completion bash > $(brew --prefix)/etc/bash_completion.d/kube-state-metrics + +You will need to start a new shell for this setup to take effect. + +Usage: + kube-state-metrics completion bash + +Flags: + --no-descriptions disable completion descriptions +` + +const autoloadFish = `Generate the autocompletion script for the fish shell. + +To load completions in your current shell session: + + kube-state-metrics completion fish | source + +To load completions for every new session, execute once: + + kube-state-metrics completion fish > ~/.config/fish/completions/kube-state-metrics.fish + +You will need to start a new shell for this setup to take effect. + +Usage: + kube-state-metrics completion fish [flags] + +Flags: + --no-descriptions disable completion descriptions +` + +// FetchLoadInstructions returns instructions for enabling autocompletion for a particular shell. +func FetchLoadInstructions(shell string) string { + switch shell { + case "zsh": + return autoloadZsh + case "bash": + return autoloadBash + case "fish": + return autoloadFish + default: + return "" + } +} + +var completionCommand = &cobra.Command{ + Use: "completion [bash|zsh|fish]", + Short: "Generate completion script for kube-state-metrics.", + DisableFlagsInUseLine: true, + Aliases: []string{"comp", "c"}, + ValidArgs: []string{"bash", "zsh", "fish"}, + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + switch args[0] { + case "bash": + _ = cmd.Root().GenBashCompletion(os.Stdout) + case "zsh": + _ = cmd.Root().GenZshCompletion(os.Stdout) + case "fish": + _ = cmd.Root().GenFishCompletion(os.Stdout, true) + } + klog.FlushAndExit(klog.ExitFlushTimeout, 0) + }, + Example: "kube-state-metrics completion bash > /tmp/kube-state-metrics.bash && source /tmp/kube-state-metrics.bash # for shells compatible with bash", +} + +// InitCommand defines the root command that others will latch onto. +var InitCommand = &cobra.Command{ + Use: "kube-state-metrics", + Short: "Add-on agent to generate and expose cluster-level metrics.", + Long: "kube-state-metrics is a simple service that listens to the Kubernetes API server and generates metrics about the state of the objects.", + Args: cobra.NoArgs, +} diff --git a/pkg/options/options.go b/pkg/options/options.go index d182fbad62..0f55785e96 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -20,94 +20,154 @@ import ( "flag" "fmt" "os" + "strings" - "k8s.io/klog" - - "github.com/spf13/pflag" + "github.com/prometheus/common/version" + "github.com/spf13/cobra" + "k8s.io/klog/v2" ) // Options are the configurable parameters for kube-state-metrics. type Options struct { - Apiserver string - Kubeconfig string - Help bool - Port int - Host string - TelemetryPort int - TelemetryHost string - Collectors CollectorSet - Namespaces NamespaceList - Shard int32 - TotalShards int - Pod string - Namespace string - MetricBlacklist MetricSet - MetricWhitelist MetricSet - Version bool - DisablePodNonGenericResourceMetrics bool - DisableNodeNonGenericResourceMetrics bool - - EnableGZIPEncoding bool - - flags *pflag.FlagSet + AnnotationsAllowList LabelsAllowList `yaml:"annotations_allow_list"` + Apiserver string `yaml:"apiserver"` + CustomResourceConfig string `yaml:"custom_resource_config"` + CustomResourceConfigFile string `yaml:"custom_resource_config_file"` + CustomResourcesOnly bool `yaml:"custom_resources_only"` + EnableGZIPEncoding bool `yaml:"enable_gzip_encoding"` + Help bool `yaml:"help"` + Host string `yaml:"host"` + Kubeconfig string `yaml:"kubeconfig"` + LabelsAllowList LabelsAllowList `yaml:"labels_allow_list"` + MetricAllowlist MetricSet `yaml:"metric_allowlist"` + MetricDenylist MetricSet `yaml:"metric_denylist"` + MetricOptInList MetricSet `yaml:"metric_opt_in_list"` + Namespace string `yaml:"namespace"` + Namespaces NamespaceList `yaml:"namespaces"` + NamespacesDenylist NamespaceList `yaml:"namespaces_denylist"` + Node NodeType `yaml:"node"` + Pod string `yaml:"pod"` + Port int `yaml:"port"` + Resources ResourceSet `yaml:"resources"` + Shard int32 `yaml:"shard"` + TLSConfig string `yaml:"tls_config"` + TelemetryHost string `yaml:"telemetry_host"` + TelemetryPort int `yaml:"telemetry_port"` + TotalShards int `yaml:"total_shards"` + UseAPIServerCache bool `yaml:"use_api_server_cache"` + + Config string + + cmd *cobra.Command +} + +// GetConfigFile is the getter for --config value. +func GetConfigFile(opt Options) string { + return opt.Config } // NewOptions returns a new instance of `Options`. func NewOptions() *Options { return &Options{ - Collectors: CollectorSet{}, - MetricWhitelist: MetricSet{}, - MetricBlacklist: MetricSet{}, + Resources: ResourceSet{}, + MetricAllowlist: MetricSet{}, + MetricDenylist: MetricSet{}, + MetricOptInList: MetricSet{}, + AnnotationsAllowList: LabelsAllowList{}, + LabelsAllowList: LabelsAllowList{}, } } // AddFlags populated the Options struct from the command line arguments passed. -func (o *Options) AddFlags() { - o.flags = pflag.NewFlagSet("", pflag.ExitOnError) - // add klog flags - klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) - klog.InitFlags(klogFlags) - o.flags.AddGoFlagSet(klogFlags) - o.flags.Lookup("logtostderr").Value.Set("true") - o.flags.Lookup("logtostderr").DefValue = "true" - o.flags.Lookup("logtostderr").NoOptDefVal = "true" - - o.flags.Usage = func() { - fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0]) - o.flags.PrintDefaults() +func (o *Options) AddFlags(cmd *cobra.Command) { + o.cmd = cmd + + completionCommand.SetHelpFunc(func(cmd *cobra.Command, args []string) { + if shellPath, ok := os.LookupEnv("SHELL"); ok { + shell := shellPath[strings.LastIndex(shellPath, "/")+1:] + fmt.Println(FetchLoadInstructions(shell)) + } else { + fmt.Println("SHELL environment variable not set, falling back to bash") + fmt.Println(FetchLoadInstructions("bash")) + } + klog.FlushAndExit(klog.ExitFlushTimeout, 0) + }) + + versionCommand := &cobra.Command{ + Use: "version", + Short: "Print version information.", + Run: func(cmd *cobra.Command, args []string) { + fmt.Printf("%s\n", version.Print("kube-state-metrics")) + klog.FlushAndExit(klog.ExitFlushTimeout, 0) + }, + } + + cmd.AddCommand(completionCommand, versionCommand) + + o.cmd.Flags().Usage = func() { + _, _ = fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0]) + o.cmd.Flags().PrintDefaults() } - o.flags.StringVar(&o.Apiserver, "apiserver", "", `The URL of the apiserver to use as a master`) - o.flags.StringVar(&o.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig file") - o.flags.BoolVarP(&o.Help, "help", "h", false, "Print Help text") - o.flags.IntVar(&o.Port, "port", 80, `Port to expose metrics on.`) - o.flags.StringVar(&o.Host, "host", "0.0.0.0", `Host to expose metrics on.`) - o.flags.IntVar(&o.TelemetryPort, "telemetry-port", 81, `Port to expose kube-state-metrics self metrics on.`) - o.flags.StringVar(&o.TelemetryHost, "telemetry-host", "0.0.0.0", `Host to expose kube-state-metrics self metrics on.`) - o.flags.Var(&o.Collectors, "collectors", fmt.Sprintf("Comma-separated list of collectors to be enabled. Defaults to %q", &DefaultCollectors)) - o.flags.Var(&o.Namespaces, "namespace", fmt.Sprintf("Comma-separated list of namespaces to be enabled. Defaults to %q", &DefaultNamespaces)) - o.flags.Var(&o.MetricWhitelist, "metric-whitelist", "Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or regex patterns. The whitelist and blacklist are mutually exclusive.") - o.flags.Var(&o.MetricBlacklist, "metric-blacklist", "Comma-separated list of metrics not to be enabled. This list comprises of exact metric names and/or regex patterns. The whitelist and blacklist are mutually exclusive.") - o.flags.Int32Var(&o.Shard, "shard", int32(0), "The instances shard nominal (zero indexed) within the total number of shards. (default 0)") - o.flags.IntVar(&o.TotalShards, "total-shards", 1, "The total number of shards. Sharding is disabled when total shards is set to 1.") + klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) + klog.InitFlags(klogFlags) + o.cmd.Flags().AddGoFlagSet(klogFlags) + _ = o.cmd.Flags().Lookup("logtostderr").Value.Set("true") + o.cmd.Flags().Lookup("logtostderr").DefValue = "true" + o.cmd.Flags().Lookup("logtostderr").NoOptDefVal = "true" autoshardingNotice := "When set, it is expected that --pod and --pod-namespace are both set. Most likely this should be passed via the downward API. This is used for auto-detecting sharding. If set, this has preference over statically configured sharding. This is experimental, it may be removed without notice." - o.flags.StringVar(&o.Pod, "pod", "", "Name of the pod that contains the kube-state-metrics container. "+autoshardingNotice) - o.flags.StringVar(&o.Namespace, "pod-namespace", "", "Name of the namespace of the pod specified by --pod. "+autoshardingNotice) - o.flags.BoolVarP(&o.Version, "version", "", false, "kube-state-metrics build version information") - o.flags.BoolVarP(&o.DisablePodNonGenericResourceMetrics, "disable-pod-non-generic-resource-metrics", "", false, "Disable pod non generic resource request and limit metrics") - o.flags.BoolVarP(&o.DisableNodeNonGenericResourceMetrics, "disable-node-non-generic-resource-metrics", "", false, "Disable node non generic resource request and limit metrics") - o.flags.BoolVar(&o.EnableGZIPEncoding, "enable-gzip-encoding", false, "Gzip responses when requested by clients via 'Accept-Encoding: gzip' header.") + o.cmd.Flags().BoolVar(&o.CustomResourcesOnly, "custom-resource-state-only", false, "Only provide Custom Resource State metrics (experimental)") + o.cmd.Flags().BoolVar(&o.EnableGZIPEncoding, "enable-gzip-encoding", false, "Gzip responses when requested by clients via 'Accept-Encoding: gzip' header.") + o.cmd.Flags().BoolVarP(&o.Help, "help", "h", false, "Print Help text") + o.cmd.Flags().BoolVarP(&o.UseAPIServerCache, "use-apiserver-cache", "", false, "Sets resourceVersion=0 for ListWatch requests, using cached resources from the apiserver instead of an etcd quorum read.") + o.cmd.Flags().Int32Var(&o.Shard, "shard", int32(0), "The instances shard nominal (zero indexed) within the total number of shards. (default 0)") + o.cmd.Flags().IntVar(&o.Port, "port", 8080, `Port to expose metrics on.`) + o.cmd.Flags().IntVar(&o.TelemetryPort, "telemetry-port", 8081, `Port to expose kube-state-metrics self metrics on.`) + o.cmd.Flags().IntVar(&o.TotalShards, "total-shards", 1, "The total number of shards. Sharding is disabled when total shards is set to 1.") + o.cmd.Flags().StringVar(&o.Apiserver, "apiserver", "", `The URL of the apiserver to use as a master`) + o.cmd.Flags().StringVar(&o.CustomResourceConfig, "custom-resource-state-config", "", "Inline Custom Resource State Metrics config YAML (experimental)") + o.cmd.Flags().StringVar(&o.CustomResourceConfigFile, "custom-resource-state-config-file", "", "Path to a Custom Resource State Metrics config file (experimental)") + o.cmd.Flags().StringVar(&o.Host, "host", "::", `Host to expose metrics on.`) + o.cmd.Flags().StringVar(&o.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig file") + o.cmd.Flags().StringVar(&o.Namespace, "pod-namespace", "", "Name of the namespace of the pod specified by --pod. "+autoshardingNotice) + o.cmd.Flags().StringVar(&o.Pod, "pod", "", "Name of the pod that contains the kube-state-metrics container. "+autoshardingNotice) + o.cmd.Flags().StringVar(&o.TLSConfig, "tls-config", "", "Path to the TLS configuration file") + o.cmd.Flags().StringVar(&o.TelemetryHost, "telemetry-host", "::", `Host to expose kube-state-metrics self metrics on.`) + o.cmd.Flags().StringVar(&o.Config, "config", "", "Path to the kube-state-metrics options config file") + o.cmd.Flags().StringVar((*string)(&o.Node), "node", "", "Name of the node that contains the kube-state-metrics pod. Most likely it should be passed via the downward API. This is used for daemonset sharding. Only available for resources (pod metrics) that support spec.nodeName fieldSelector. This is experimental.") + o.cmd.Flags().Var(&o.AnnotationsAllowList, "metric-annotations-allowlist", "Comma-separated list of Kubernetes annotations keys that will be used in the resource' labels metric. By default the metric contains only name and namespace labels. To include additional annotations provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]').") + o.cmd.Flags().Var(&o.LabelsAllowList, "metric-labels-allowlist", "Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the metric contains only name and namespace labels. To include additional labels provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided as a key, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'.") + o.cmd.Flags().Var(&o.MetricAllowlist, "metric-allowlist", "Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive.") + o.cmd.Flags().Var(&o.MetricDenylist, "metric-denylist", "Comma-separated list of metrics not to be enabled. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive.") + o.cmd.Flags().Var(&o.MetricOptInList, "metric-opt-in-list", "Comma-separated list of metrics which are opt-in and not enabled by default. This is in addition to the metric allow- and denylists") + o.cmd.Flags().Var(&o.Namespaces, "namespaces", fmt.Sprintf("Comma-separated list of namespaces to be enabled. Defaults to %q", &DefaultNamespaces)) + o.cmd.Flags().Var(&o.NamespacesDenylist, "namespaces-denylist", "Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, only namespaces that are excluded in namespaces-denylist will be used.") + o.cmd.Flags().Var(&o.Resources, "resources", fmt.Sprintf("Comma-separated list of Resources to be enabled. Defaults to %q", &DefaultResources)) } // Parse parses the flag definitions from the argument list. func (o *Options) Parse() error { - err := o.flags.Parse(os.Args) + err := o.cmd.Execute() return err } // Usage is the function called when an error occurs while parsing flags. func (o *Options) Usage() { - o.flags.Usage() + _ = o.cmd.Flags().FlagUsages() +} + +// Validate validates arguments +func (o *Options) Validate() error { + shardableResource := "pods" + if o.Node == "" { + return nil + } + for _, x := range o.Resources.AsSlice() { + if x != shardableResource { + return fmt.Errorf("resource %s can't be sharded by field selector spec.nodeName", x) + } + } + return nil } diff --git a/pkg/options/options_test.go b/pkg/options/options_test.go index f0e63064e1..0e17c0c5a8 100644 --- a/pkg/options/options_test.go +++ b/pkg/options/options_test.go @@ -18,58 +18,48 @@ package options import ( "os" - "sync" "testing" - - "github.com/spf13/pflag" ) func TestOptionsParse(t *testing.T) { tests := []struct { - Desc string - Args []string - RecoverInvoked bool + Desc string + Args []string + ExpectsError bool }{ { - Desc: "collectors command line argument", - Args: []string{"./kube-state-metrics", "--collectors=configmaps,pods"}, - RecoverInvoked: false, + Desc: "resources command line argument", + Args: []string{"./kube-state-metrics", "--resources=configmaps,pods"}, + ExpectsError: false, + }, + { + Desc: "namespaces command line argument", + Args: []string{"./kube-state-metrics", "--namespaces=default,kube-system"}, + ExpectsError: false, }, { - Desc: "namespace command line argument", - Args: []string{"./kube-state-metrics", "--namespace=default,kube-system"}, - RecoverInvoked: false, + Desc: "foo command line argument", + Args: []string{"./kube-state-metrics", "--foo=bar,baz"}, + ExpectsError: true, }, } - for _, test := range tests { - var wg sync.WaitGroup - - opts := NewOptions() - opts.AddFlags() - - flags := pflag.NewFlagSet("options_test", pflag.PanicOnError) - flags.AddFlagSet(opts.flags) + opts := NewOptions() + opts.AddFlags(InitCommand) - opts.flags = flags - - os.Args = test.Args + for _, test := range tests { + t.Run(test.Desc, func(t *testing.T) { + os.Args = test.Args - wg.Add(1) - go func() { - defer wg.Done() - defer func() { - if err := recover(); err != nil { - test.RecoverInvoked = true - } - }() + err := opts.Parse() - opts.Parse() - }() + if !test.ExpectsError && err != nil { + t.Errorf("Error for test with description: %s: %v", test.Desc, err.Error()) + } - wg.Wait() - if test.RecoverInvoked { - t.Errorf("Test error for Desc: %s. Test panic", test.Desc) - } + if test.ExpectsError && err == nil { + t.Errorf("Expected error for test with description: %s", test.Desc) + } + }) } } diff --git a/pkg/options/collector.go b/pkg/options/resource.go similarity index 92% rename from pkg/options/collector.go rename to pkg/options/resource.go index f3afaf76c1..09e0e0400c 100644 --- a/pkg/options/collector.go +++ b/pkg/options/resource.go @@ -24,8 +24,8 @@ var ( // DefaultNamespaces is the default namespace selector for selecting and filtering across all namespaces. DefaultNamespaces = NamespaceList{metav1.NamespaceAll} - // DefaultCollectors represents the default set of collectors in kube-state-metrics. - DefaultCollectors = CollectorSet{ + // DefaultResources represents the default set of resources in kube-state-metrics. + DefaultResources = ResourceSet{ "certificatesigningrequests": struct{}{}, "configmaps": struct{}{}, "cronjobs": struct{}{}, @@ -35,6 +35,7 @@ var ( "horizontalpodautoscalers": struct{}{}, "ingresses": struct{}{}, "jobs": struct{}{}, + "leases": struct{}{}, "limitranges": struct{}{}, "mutatingwebhookconfigurations": struct{}{}, "namespaces": struct{}{}, diff --git a/pkg/options/types.go b/pkg/options/types.go index 914519f625..ac0c11275e 100644 --- a/pkg/options/types.go +++ b/pkg/options/types.go @@ -17,12 +17,19 @@ limitations under the License. package options import ( + "errors" "sort" "strings" + "k8s.io/apimachinery/pkg/fields" + + "k8s.io/klog/v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +var errLabelsAllowListFormat = errors.New("invalid format, metric=[label1,label2,labeln...],metricN=[]") + // MetricSet represents a collection which has a unique set of metrics. type MetricSet map[string]struct{} @@ -60,19 +67,19 @@ func (ms *MetricSet) Type() string { return "string" } -// CollectorSet represents a collection which has a unique set of collectors. -type CollectorSet map[string]struct{} +// ResourceSet represents a collection which has a unique set of resources. +type ResourceSet map[string]struct{} -func (c *CollectorSet) String() string { - s := *c +func (r *ResourceSet) String() string { + s := *r ss := s.AsSlice() sort.Strings(ss) return strings.Join(ss, ",") } -// Set converts a comma-separated string of collectors into a slice and appends it to the CollectorSet. -func (c *CollectorSet) Set(value string) error { - s := *c +// Set converts a comma-separated string of resources into a slice and appends it to the ResourceSet. +func (r *ResourceSet) Set(value string) error { + s := *r cols := strings.Split(value, ",") for _, col := range cols { col = strings.TrimSpace(col) @@ -83,20 +90,68 @@ func (c *CollectorSet) Set(value string) error { return nil } -// AsSlice returns the Collector in the form of a plain string slice. -func (c CollectorSet) AsSlice() []string { - cols := make([]string, 0, len(c)) - for col := range c { +// AsSlice returns the Resource in the form of a plain string slice. +func (r ResourceSet) AsSlice() []string { + cols := make([]string, 0, len(r)) + for col := range r { cols = append(cols, col) } return cols } -// Type returns a descriptive string about the CollectorSet type. -func (c *CollectorSet) Type() string { +// Type returns a descriptive string about the ResourceSet type. +func (r *ResourceSet) Type() string { return "string" } +// NodeType represents a nodeName to query from. +type NodeType string + +// GetNodeFieldSelector returns a nodename field selector. +func (n *NodeType) GetNodeFieldSelector() string { + if string(*n) != "" { + return fields.OneTermEqualSelector("spec.nodeName", string(*n)).String() + } + return EmptyFieldSelector() +} + +// EmptyFieldSelector returns an empty field selector. +func EmptyFieldSelector() string { + return fields.Nothing().String() +} + +// MergeFieldSelectors returns AND of a list of field selectors. +func MergeFieldSelectors(selectors []string) (string, error) { + var err error + merged := EmptyFieldSelector() + for _, s := range selectors { + merged, err = MergeTwoFieldSelectors(merged, s) + if err != nil { + return "", err + } + } + return merged, nil +} + +// MergeTwoFieldSelectors returns AND of two field selectors. +func MergeTwoFieldSelectors(s1 string, s2 string) (string, error) { + selector1, err := fields.ParseSelector(s1) + if err != nil { + return EmptyFieldSelector(), err + } + selector2, err := fields.ParseSelector(s2) + if err != nil { + return EmptyFieldSelector(), err + } + if selector1.Empty() { + return selector2.String(), nil + } + if selector2.Empty() { + return selector1.String(), nil + } + return fields.AndSelectors(selector1, selector2).String(), nil +} + // NamespaceList represents a list of namespaces to query from. type NamespaceList []string @@ -122,7 +177,130 @@ func (n *NamespaceList) Set(value string) error { return nil } +// GetNamespaces is a helper function to get namespaces from opts.Namespaces +func (n *NamespaceList) GetNamespaces() NamespaceList { + ns := *n + if len(*n) == 0 { + klog.InfoS("Using all namespaces") + ns = DefaultNamespaces + } else { + if n.IsAllNamespaces() { + klog.InfoS("Using all namespaces") + } else { + klog.InfoS("Using namespaces", "nameSpaces", ns) + } + } + return ns +} + +// GetExcludeNSFieldSelector will return excluded namespace field selector +// if nsDenylist = {case1,case2}, the result will be "metadata.namespace!=case1,metadata.namespace!=case2". +func (n *NamespaceList) GetExcludeNSFieldSelector(nsDenylist []string) string { + if len(nsDenylist) == 0 { + return "" + } + + namespaceExcludeSelectors := make([]fields.Selector, len(nsDenylist)) + for i, ns := range nsDenylist { + selector := fields.OneTermNotEqualSelector("metadata.namespace", ns) + namespaceExcludeSelectors[i] = selector + } + return fields.AndSelectors(namespaceExcludeSelectors...).String() +} + // Type returns a descriptive string about the NamespaceList type. func (n *NamespaceList) Type() string { return "string" } + +// LabelWildcard allowlists any label +const LabelWildcard = "*" + +// LabelsAllowList represents a list of allowed labels for metrics. +type LabelsAllowList map[string][]string + +// Set converts a comma-separated string of resources and their allowed Kubernetes labels and appends to the LabelsAllowList. +// Value is in the following format: +// resource=[k8s-label-name,another-k8s-label],another-resource[k8s-label] +// Example: pods=[app.kubernetes.io/component,app],resource=[blah] +func (l *LabelsAllowList) Set(value string) error { + // Taken from text/scanner EOF constant. + const EOF = -1 + var ( + m = make(map[string][]string, len(*l)) + previous rune + next rune + firstWordPos int + name string + ) + firstWordPos = 0 + + for i, v := range value { + if i+1 == len(value) { + next = EOF + } else { + next = []rune(value)[i+1] + } + if i-1 >= 0 { + previous = []rune(value)[i-1] + } else { + previous = v + } + + switch v { + case '=': + if previous == ',' || next != '[' { + return errLabelsAllowListFormat + } + name = strings.TrimSpace(string(([]rune(value)[firstWordPos:i]))) + m[name] = []string{} + firstWordPos = i + 1 + case '[': + if previous != '=' { + return errLabelsAllowListFormat + } + firstWordPos = i + 1 + case ']': + // if after metric group, has char not comma or end. + if next != EOF && next != ',' { + return errLabelsAllowListFormat + } + if previous != '[' { + m[name] = append(m[name], strings.TrimSpace(string(([]rune(value)[firstWordPos:i])))) + } + firstWordPos = i + 1 + case ',': + // if starts or ends with comma + if previous == v || next == EOF || next == ']' { + return errLabelsAllowListFormat + } + if previous != ']' { + m[name] = append(m[name], strings.TrimSpace(string(([]rune(value)[firstWordPos:i])))) + } + firstWordPos = i + 1 + } + } + *l = m + return nil +} + +// asSlice returns the LabelsAllowList in the form of plain string slice. +func (l LabelsAllowList) asSlice() []string { + metrics := make([]string, 0, len(l)) + for metric := range l { + metrics = append(metrics, metric) + } + return metrics +} + +func (l *LabelsAllowList) String() string { + s := *l + ss := s.asSlice() + sort.Strings(ss) + return strings.Join(ss, ",") +} + +// Type returns a descriptive string about the LabelsAllowList type. +func (l *LabelsAllowList) Type() string { + return "string" +} diff --git a/pkg/options/types_test.go b/pkg/options/types_test.go index 53b3400a49..3ccaf4c0e4 100644 --- a/pkg/options/types_test.go +++ b/pkg/options/types_test.go @@ -21,23 +21,23 @@ import ( "testing" ) -func TestCollectorSetSet(t *testing.T) { +func TestResourceSetSet(t *testing.T) { tests := []struct { Desc string Value string - Wanted CollectorSet + Wanted ResourceSet WantedError bool }{ { - Desc: "empty collectors", + Desc: "empty resources", Value: "", - Wanted: CollectorSet{}, + Wanted: ResourceSet{}, WantedError: false, }, { - Desc: "normal collectors", + Desc: "normal resources", Value: "configmaps,cronjobs,daemonsets,deployments", - Wanted: CollectorSet(map[string]struct{}{ + Wanted: ResourceSet(map[string]struct{}{ "configmaps": {}, "cronjobs": {}, "daemonsets": {}, @@ -48,7 +48,7 @@ func TestCollectorSetSet(t *testing.T) { } for _, test := range tests { - cs := &CollectorSet{} + cs := &ResourceSet{} gotError := cs.Set(test.Value) if !(((gotError == nil && !test.WantedError) || (gotError != nil && test.WantedError)) && reflect.DeepEqual(*cs, test.Wanted)) { t.Errorf("Test error for Desc: %s. Want: %+v. Got: %+v. Wanted Error: %v, Got Error: %v", test.Desc, test.Wanted, *cs, test.WantedError, gotError) @@ -86,6 +86,169 @@ func TestNamespaceListSet(t *testing.T) { } } +func TestNamespaceList_GetNamespaces(t *testing.T) { + tests := []struct { + Desc string + Namespaces NamespaceList + Wanted NamespaceList + }{ + { + Desc: "empty DeniedNamespaces", + Namespaces: NamespaceList{}, + Wanted: NamespaceList{""}, + }, + { + Desc: "all DeniedNamespaces", + Namespaces: DefaultNamespaces, + Wanted: NamespaceList{""}, + }, + { + Desc: "general namespaceDenylist", + Namespaces: NamespaceList{"default", "kube-system"}, + Wanted: NamespaceList{"default", "kube-system"}, + }, + } + + for _, test := range tests { + ns := &test.Namespaces + allowedNamespaces := ns.GetNamespaces() + if !reflect.DeepEqual(allowedNamespaces, test.Wanted) { + t.Errorf("Test error for Desc: %s. Want: %+v. Got: %+v.", test.Desc, test.Wanted, allowedNamespaces) + } + } +} + +func TestNamespaceList_ExcludeNamespacesFieldSelector(t *testing.T) { + tests := []struct { + Desc string + Namespaces NamespaceList + DeniedNamespaces NamespaceList + Wanted string + }{ + { + Desc: "empty DeniedNamespaces", + Namespaces: NamespaceList{"default", "kube-system"}, + DeniedNamespaces: NamespaceList{}, + Wanted: "", + }, + { + Desc: "all DeniedNamespaces", + Namespaces: DefaultNamespaces, + DeniedNamespaces: NamespaceList{"some-system"}, + Wanted: "metadata.namespace!=some-system", + }, + { + Desc: "general case", + Namespaces: DefaultNamespaces, + DeniedNamespaces: NamespaceList{"case1-system", "case2-system"}, + Wanted: "metadata.namespace!=case1-system,metadata.namespace!=case2-system", + }, + } + + for _, test := range tests { + ns := test.Namespaces + deniedNS := test.DeniedNamespaces + actual := ns.GetExcludeNSFieldSelector(deniedNS) + if !reflect.DeepEqual(actual, test.Wanted) { + t.Errorf("Test error for Desc: %s. Want: %+v. Got: %+v.", test.Desc, test.Wanted, actual) + } + } +} + +func TestNodeFieldSelector(t *testing.T) { + tests := []struct { + Desc string + Node NodeType + Wanted string + }{ + { + Desc: "empty node name", + Node: "", + Wanted: "", + }, + { + Desc: "with node name", + Node: "k8s-node-1", + Wanted: "spec.nodeName=k8s-node-1", + }, + } + + for _, test := range tests { + node := test.Node + actual := node.GetNodeFieldSelector() + if !reflect.DeepEqual(actual, test.Wanted) { + t.Errorf("Test error for Desc: %s. Want: %+v. Got: %+v.", test.Desc, test.Wanted, actual) + } + } +} + +func TestMergeFieldSelectors(t *testing.T) { + tests := []struct { + Desc string + Namespaces NamespaceList + DeniedNamespaces NamespaceList + Node NodeType + Wanted string + }{ + { + Desc: "empty DeniedNamespaces", + Namespaces: NamespaceList{"default", "kube-system"}, + DeniedNamespaces: NamespaceList{}, + Node: "", + Wanted: "", + }, + { + Desc: "all DeniedNamespaces", + Namespaces: DefaultNamespaces, + DeniedNamespaces: NamespaceList{"some-system"}, + Node: "", + Wanted: "metadata.namespace!=some-system", + }, + { + Desc: "general case", + Namespaces: DefaultNamespaces, + DeniedNamespaces: NamespaceList{"case1-system", "case2-system"}, + Node: "", + Wanted: "metadata.namespace!=case1-system,metadata.namespace!=case2-system", + }, + { + Desc: "empty DeniedNamespaces", + Namespaces: NamespaceList{"default", "kube-system"}, + DeniedNamespaces: NamespaceList{}, + Node: "k8s-node-1", + Wanted: "spec.nodeName=k8s-node-1", + }, + { + Desc: "all DeniedNamespaces", + Namespaces: DefaultNamespaces, + DeniedNamespaces: NamespaceList{"some-system"}, + Node: "k8s-node-1", + Wanted: "metadata.namespace!=some-system,spec.nodeName=k8s-node-1", + }, + { + Desc: "general case", + Namespaces: DefaultNamespaces, + DeniedNamespaces: NamespaceList{"case1-system", "case2-system"}, + Node: "k8s-node-1", + Wanted: "metadata.namespace!=case1-system,metadata.namespace!=case2-system,spec.nodeName=k8s-node-1", + }, + } + + for _, test := range tests { + ns := test.Namespaces + deniedNS := test.DeniedNamespaces + selector1 := ns.GetExcludeNSFieldSelector(deniedNS) + selector2 := test.Node.GetNodeFieldSelector() + actual, err := MergeFieldSelectors([]string{selector1, selector2}) + if err != nil { + t.Errorf("Test error for Desc: %s. Can't merge field selector %v.", test.Desc, err) + } + if !reflect.DeepEqual(actual, test.Wanted) { + t.Errorf("Test error for Desc: %s. Want: %+v. Got: %+v.", test.Desc, test.Wanted, actual) + } + } +} + func TestMetricSetSet(t *testing.T) { tests := []struct { Desc string @@ -106,6 +269,14 @@ func TestMetricSetSet(t *testing.T) { "kube_daemonset_labels": {}, }), }, + { + Desc: "newlines are ignored", + Value: "\n^kube_.+_annotations$,\n ^kube_secret_labels$\n", + Wanted: MetricSet{ + "^kube_secret_labels$": struct{}{}, + "^kube_.+_annotations$": struct{}{}, + }, + }, } for _, test := range tests { @@ -116,3 +287,108 @@ func TestMetricSetSet(t *testing.T) { } } } + +func TestLabelsAllowListSet(t *testing.T) { + tests := []struct { + Desc string + Value string + Wanted LabelsAllowList + err bool + }{ + { + Desc: "empty labels list", + Value: "", + Wanted: LabelsAllowList{}, + }, + { + Desc: "[invalid] space delimited", + Value: "cronjobs=[somelabel,label2] cronjobs=[label3,label4]", + Wanted: LabelsAllowList(map[string][]string{}), + err: true, + }, + { + Desc: "[invalid] normal missing bracket", + Value: "cronjobs=[somelabel,label2],cronjobs=label3,label4]", + Wanted: LabelsAllowList(map[string][]string{}), + err: true, + }, + + { + Desc: "[invalid] no comma between metrics", + Value: "cronjobs=[somelabel,label2]cronjobs=[label3,label4]", + Wanted: LabelsAllowList(map[string][]string{}), + err: true, + }, + { + Desc: "[invalid] no '=' between name and label list", + Value: "cronjobs[somelabel,label2]cronjobs=[label3,label4]", + Wanted: LabelsAllowList(map[string][]string{}), + err: true, + }, + { + Desc: "one resource", + Value: "cronjobs=[somelabel.io,label2/blah]", + Wanted: LabelsAllowList(map[string][]string{ + "cronjobs": { + "somelabel.io", + "label2/blah", + }}), + }, + { + Desc: "two resources", + Value: "pods=[podsone,pods-two],nodes=[nodesone,nodestwo],namespaces=[nsone,nstwo]", + Wanted: LabelsAllowList(map[string][]string{ + "pods": { + "podsone", + "pods-two"}, + "nodes": { + "nodesone", + "nodestwo"}, + "namespaces": { + "nsone", + "nstwo"}}), + }, + { + Desc: "with empty allow labels", + Value: "cronjobs=[somelabel,label2],pods=[]", + Wanted: LabelsAllowList(map[string][]string{ + "cronjobs": { + "somelabel", + "label2", + }, + "pods": {}}), + }, + { + Desc: "with wildcard", + Value: "cronjobs=[*],pods=[*,foo],namespaces=[bar,*]", + Wanted: LabelsAllowList(map[string][]string{ + "cronjobs": { + "*", + }, + "pods": { + "*", + "foo", + }, + "namespaces": { + "bar", + "*"}}), + }, + { + Desc: "with key as wildcard", + Value: "*=[*]", + Wanted: LabelsAllowList(map[string][]string{ + "*": { + "*", + }, + }), + }, + } + + for _, test := range tests { + lal := &LabelsAllowList{} + gotError := lal.Set(test.Value) + if gotError != nil && !test.err || !reflect.DeepEqual(*lal, test.Wanted) { + t.Errorf("Test error for Desc: %s\n Want: \n%+v\n Got: \n%#+v\n Got Error: %#v", test.Desc, test.Wanted, *lal, gotError) + } + } +} diff --git a/pkg/sharding/listwatch.go b/pkg/sharding/listwatch.go index 688c69bda7..9f83c3452f 100644 --- a/pkg/sharding/listwatch.go +++ b/pkg/sharding/listwatch.go @@ -32,6 +32,8 @@ type shardedListWatch struct { lw cache.ListerWatcher } +// NewShardedListWatch returns a new shardedListWatch via the cache.ListerWatcher interface. +// In the case of no sharding needed, it returns the provided cache.ListerWatcher func NewShardedListWatch(shard int32, totalShards int, lw cache.ListerWatcher) cache.ListerWatcher { // This is an "optimization" as this configuration means no sharding is to // be performed. @@ -51,6 +53,10 @@ func (s *shardedListWatch) List(options metav1.ListOptions) (runtime.Object, err if err != nil { return nil, err } + metaObj, err := meta.ListAccessor(list) + if err != nil { + return nil, err + } res := &metav1.List{ Items: []runtime.RawExtension{}, } @@ -63,6 +69,7 @@ func (s *shardedListWatch) List(options metav1.ListOptions) (runtime.Object, err res.Items = append(res.Items, runtime.RawExtension{Object: item}) } } + res.ListMeta.ResourceVersion = metaObj.GetResourceVersion() return res, nil } diff --git a/pkg/sharding/metrics.go b/pkg/sharding/metrics.go new file mode 100644 index 0000000000..c984739603 --- /dev/null +++ b/pkg/sharding/metrics.go @@ -0,0 +1,53 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sharding + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +const ( + // LabelOrdinal is name of Prometheus metric label to use in conjunction with kube_state_metrics_shard_ordinal. + LabelOrdinal = "shard_ordinal" +) + +// Metrics stores the pointers of kube_state_metrics_shard_ordinal +// and kube_state_metrics_total_shards metrics. +type Metrics struct { + Ordinal *prometheus.GaugeVec + Total prometheus.Gauge +} + +// NewShardingMetrics takes in a prometheus registry and initializes +// and registers sharding configuration metrics. It returns those registered metrics. +func NewShardingMetrics(r prometheus.Registerer) *Metrics { + return &Metrics{ + Ordinal: promauto.With(r).NewGaugeVec( + prometheus.GaugeOpts{ + Name: "kube_state_metrics_shard_ordinal", + Help: "Current sharding ordinal/index of this instance", + }, []string{LabelOrdinal}, + ), + Total: promauto.With(r).NewGauge( + prometheus.GaugeOpts{ + Name: "kube_state_metrics_total_shards", + Help: "Number of total shards this instance is aware of", + }, + ), + } +} diff --git a/pkg/util/proc/reaper.go b/pkg/util/proc/reaper.go index a007075e43..216af77c98 100644 --- a/pkg/util/proc/reaper.go +++ b/pkg/util/proc/reaper.go @@ -1,3 +1,4 @@ +//go:build linux // +build linux /* @@ -23,21 +24,21 @@ import ( "os/signal" "syscall" - "k8s.io/klog" + "k8s.io/klog/v2" ) // StartReaper starts a goroutine to reap processes if called from a process // that has pid 1. func StartReaper() { if os.Getpid() == 1 { - klog.V(4).Infof("Launching reaper") + klog.V(4).InfoS("Launching reaper") go func() { sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGCHLD) for { // Wait for a child to terminate sig := <-sigs - klog.V(4).Infof("Signal received: %v", sig) + klog.V(4).InfoS("Signal received", "signal", sig) for { // Reap processes cpid, _ := syscall.Wait4(-1, nil, syscall.WNOHANG, nil) @@ -45,7 +46,7 @@ func StartReaper() { break } - klog.V(4).Infof("Reaped process with pid %d", cpid) + klog.V(4).InfoS("Reaped process with pid", "cpid", cpid) } } }() diff --git a/pkg/util/proc/reaper_unsupported.go b/pkg/util/proc/reaper_unsupported.go index e404bb6bdf..8b76c8fa8c 100644 --- a/pkg/util/proc/reaper_unsupported.go +++ b/pkg/util/proc/reaper_unsupported.go @@ -1,3 +1,4 @@ +//go:build !linux // +build !linux /* diff --git a/pkg/version/version.go b/pkg/version/version.go deleted file mode 100644 index b6c42b07c3..0000000000 --- a/pkg/version/version.go +++ /dev/null @@ -1,64 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package version - -import ( - "fmt" - "os" - "path/filepath" - "runtime" -) - -var ( - // Release returns the release version - Release = "UNKNOWN" - // Commit returns the short sha from git - Commit = "UNKNOWN" - // BuildDate is the build date - BuildDate = "" -) - -// Version is the current version of kube-state-metrics. -// Update this whenever making a new release. -// The version is of the format Major.Minor.Patch -// -// Increment major number for new feature additions and behavioral changes. -// Increment minor number for bug fixes and performance enhancements. -// Increment patch number for critical fixes to existing releases. -type Version struct { - GitCommit string - BuildDate string - Release string - GoVersion string - Compiler string - Platform string -} - -func (v Version) String() string { - return fmt.Sprintf("%s/%s (%s/%s) kube-state-metrics/%s", - filepath.Base(os.Args[0]), v.Release, - runtime.GOOS, runtime.GOARCH, v.GitCommit) -} - -// GetVersion returns the kube-state-metrics version. -func GetVersion() Version { - return Version{ - GitCommit: Commit, - BuildDate: BuildDate, - Release: Release, - GoVersion: runtime.Version(), - Compiler: runtime.Compiler, - Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), - } -} diff --git a/pkg/watch/watch.go b/pkg/watch/watch.go index 540e4a43f2..751097caaa 100644 --- a/pkg/watch/watch.go +++ b/pkg/watch/watch.go @@ -18,12 +18,14 @@ package watch import ( "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/tools/cache" ) +// ListWatchMetrics stores the pointers of kube_state_metrics_[list|watch]_total metrics. type ListWatchMetrics struct { WatchTotal *prometheus.CounterVec ListTotal *prometheus.CounterVec @@ -32,50 +34,52 @@ type ListWatchMetrics struct { // NewListWatchMetrics takes in a prometheus registry and initializes // and registers the kube_state_metrics_list_total and // kube_state_metrics_watch_total metrics. It returns those registered metrics. -func NewListWatchMetrics(r *prometheus.Registry) *ListWatchMetrics { - var m ListWatchMetrics - m.WatchTotal = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "kube_state_metrics_watch_total", - Help: "Number of total resource watches in kube-state-metrics", - }, - []string{"result", "resource"}, - ) - - m.ListTotal = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "kube_state_metrics_list_total", - Help: "Number of total resource list in kube-state-metrics", - }, - []string{"result", "resource"}, - ) - if r != nil { - r.MustRegister( - m.ListTotal, - m.WatchTotal, - ) +func NewListWatchMetrics(r prometheus.Registerer) *ListWatchMetrics { + return &ListWatchMetrics{ + WatchTotal: promauto.With(r).NewCounterVec( + prometheus.CounterOpts{ + Name: "kube_state_metrics_watch_total", + Help: "Number of total resource watches in kube-state-metrics", + }, + []string{"result", "resource"}, + ), + ListTotal: promauto.With(r).NewCounterVec( + prometheus.CounterOpts{ + Name: "kube_state_metrics_list_total", + Help: "Number of total resource list in kube-state-metrics", + }, + []string{"result", "resource"}, + ), } - return &m } +// InstrumentedListerWatcher provides the kube_state_metrics_watch_total metric +// with a cache.ListerWatcher obj and the related resource. type InstrumentedListerWatcher struct { - lw cache.ListerWatcher - metrics *ListWatchMetrics - resource string + lw cache.ListerWatcher + metrics *ListWatchMetrics + resource string + useAPIServerCache bool } // NewInstrumentedListerWatcher returns a new InstrumentedListerWatcher. -func NewInstrumentedListerWatcher(lw cache.ListerWatcher, metrics *ListWatchMetrics, resource string) cache.ListerWatcher { +func NewInstrumentedListerWatcher(lw cache.ListerWatcher, metrics *ListWatchMetrics, resource string, useAPIServerCache bool) cache.ListerWatcher { return &InstrumentedListerWatcher{ - lw: lw, - metrics: metrics, - resource: resource, + lw: lw, + metrics: metrics, + resource: resource, + useAPIServerCache: useAPIServerCache, } } // List is a wrapper func around the cache.ListerWatcher.List func. It increases the success/error // / counters based on the outcome of the List operation it instruments. func (i *InstrumentedListerWatcher) List(options metav1.ListOptions) (res runtime.Object, err error) { + + if i.useAPIServerCache { + options.ResourceVersion = "0" + } + res, err = i.lw.List(options) if err != nil { i.metrics.ListTotal.WithLabelValues("error", i.resource).Inc() diff --git a/scripts/generate-help-text.sh b/scripts/generate-help-text.sh index 8d58828a1c..00393eac81 100755 --- a/scripts/generate-help-text.sh +++ b/scripts/generate-help-text.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash echo "$ kube-state-metrics -h" > help.txt -./kube-state-metrics -h 2>> help.txt +./kube-state-metrics -h >> help.txt exit 0 diff --git a/scripts/jsonnetfile.json b/scripts/jsonnetfile.json index 4f5849f7b2..4d38c94e91 100644 --- a/scripts/jsonnetfile.json +++ b/scripts/jsonnetfile.json @@ -1,22 +1,22 @@ { - "dependencies": [ - { - "name": "kube-state-metrics", - "source": { - "local": { - "directory": "../jsonnet/kube-state-metrics" - } - }, - "version": "" - }, - { - "name": "kube-state-metrics-mixin", - "source": { - "local": { - "directory": "../jsonnet/kube-state-metrics-mixin" - } - }, - "version": "" + "version": 1, + "dependencies": [ + { + "source": { + "local": { + "directory": "../jsonnet/kube-state-metrics" } - ] + }, + "version": "" + }, + { + "source": { + "local": { + "directory": "../jsonnet/kube-state-metrics-mixin" + } + }, + "version": "" + } + ], + "legacyImports": false } diff --git a/scripts/jsonnetfile.lock.json b/scripts/jsonnetfile.lock.json index f0d45fcff4..4d38c94e91 100644 --- a/scripts/jsonnetfile.lock.json +++ b/scripts/jsonnetfile.lock.json @@ -1,18 +1,7 @@ { + "version": 1, "dependencies": [ { - "name": "ksonnet", - "source": { - "git": { - "remote": "https://github.com/ksonnet/ksonnet-lib", - "subdir": "" - } - }, - "version": "0d2f82676817bbf9e4acf6495b2090205f323b9f", - "sum": "h28BXZ7+vczxYJ2sCt8JuR9+yznRtU/iA6DCpQUrtEg=" - }, - { - "name": "kube-state-metrics", "source": { "local": { "directory": "../jsonnet/kube-state-metrics" @@ -21,7 +10,6 @@ "version": "" }, { - "name": "kube-state-metrics-mixin", "source": { "local": { "directory": "../jsonnet/kube-state-metrics-mixin" @@ -29,5 +17,6 @@ }, "version": "" } - ] + ], + "legacyImports": false } diff --git a/scripts/standard.jsonnet b/scripts/standard.jsonnet index 8328cab4ba..0839cecf5d 100644 --- a/scripts/standard.jsonnet +++ b/scripts/standard.jsonnet @@ -5,5 +5,5 @@ ksm { name:: 'kube-state-metrics', namespace:: 'kube-system', version:: version, - image:: 'quay.io/coreos/kube-state-metrics:v' + version, + image:: 'registry.k8s.io/kube-state-metrics/kube-state-metrics:v' + version, } diff --git a/tests/README.md b/tests/README.md index 900449c25d..410daf7a78 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,16 +1,15 @@ # End2end testsuite This folder contains simple e2e tests. -When launched it spins up a kubernetes cluster using minikube, creates several kubernetes resources and launches a kube-state-metrics deployment. -Then, it downloads kube-state-metrics' metrics and examines validity using `promtool` tool. +When launched, it spins up a kubernetes cluster using [kind](https://kind.sigs.k8s.io/), creates several kubernetes resources and launches a kube-state-metrics deployment. +Then, it runs verification tests: check metrics' presence, lint metrics, check service health, etc. -The testsuite is run automatically using Travis. +The test suite is run automatically using Github Actions. ## Running locally To run the e2e tests locally run the following command: ```bash -export MINIKUBE_DRIVER=virtualbox # choose minikube's driver of your choice ./tests/e2e.sh ``` diff --git a/tests/compare_benchmarks.sh b/tests/compare_benchmarks.sh index 67cbd8aff5..af871bf9fd 100755 --- a/tests/compare_benchmarks.sh +++ b/tests/compare_benchmarks.sh @@ -38,4 +38,4 @@ echo "" echo "### Result" echo "old=${REF_TO_COMPARE} new=${REF_CURRENT}" -benchcmp "$RESULT_TO_COMPARE" "$RESULT_CURRENT" +benchstat "$RESULT_TO_COMPARE" "$RESULT_CURRENT" diff --git a/tests/e2e.sh b/tests/e2e.sh index 9507d78fa1..1dea5b283d 100755 --- a/tests/e2e.sh +++ b/tests/e2e.sh @@ -17,24 +17,25 @@ set -e set -o pipefail -KUBERNETES_VERSION=v1.17.3 +case $(uname -m) in + aarch64) ARCH="arm64";; + x86_64) ARCH="amd64";; + *) ARCH="$(uname -m)";; +esac + +NODE_IMAGE_NAME="docker.io/kindest/node" +KUBERNETES_VERSION=${KUBERNETES_VERSION:-"v1.26.0"} KUBE_STATE_METRICS_LOG_DIR=./log -KUBE_STATE_METRICS_IMAGE_NAME='quay.io/coreos/kube-state-metrics' -PROMETHEUS_VERSION=2.12.0 -E2E_SETUP_MINIKUBE=${E2E_SETUP_MINIKUBE:-} +KUBE_STATE_METRICS_CURRENT_IMAGE_NAME="registry.k8s.io/kube-state-metrics/kube-state-metrics" +KUBE_STATE_METRICS_IMAGE_NAME="registry.k8s.io/kube-state-metrics/kube-state-metrics-${ARCH}" +E2E_SETUP_KIND=${E2E_SETUP_KIND:-} E2E_SETUP_KUBECTL=${E2E_SETUP_KUBECTL:-} -E2E_SETUP_PROMTOOL=${E2E_SETUP_PROMTOOL:-} -MINIKUBE_VERSION=v1.3.1 -MINIKUBE_DRIVER=${MINIKUBE_DRIVER:-virtualbox} +KIND_VERSION=v0.17.0 SUDO=${SUDO:-} OS=$(uname -s | awk '{print tolower($0)}') OS=${OS:-linux} -EXCLUDED_RESOURCE_REGEX="verticalpodautoscaler" - -mkdir -p ${KUBE_STATE_METRICS_LOG_DIR} - function finish() { echo "calling cleanup function" # kill kubectl proxy in background @@ -43,49 +44,38 @@ function finish() { kubectl delete -f tests/manifests/ || true } -function setup_minikube() { - curl -sLo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-"${OS}"-amd64 \ - && chmod +x minikube \ - && ${SUDO} mv minikube /usr/local/bin/ +function setup_kind() { + curl -sLo kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-${OS}-${ARCH}" \ + && chmod +x kind \ + && ${SUDO} mv kind /usr/local/bin/ } function setup_kubectl() { - curl -sLo kubectl https://storage.googleapis.com/kubernetes-release/release/"$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)"/bin/"${OS}"/amd64/kubectl \ + curl -sLo kubectl https://storage.googleapis.com/kubernetes-release/release/"$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)"/bin/"${OS}"/"${ARCH}"/kubectl \ && chmod +x kubectl \ && ${SUDO} mv kubectl /usr/local/bin/ } -function setup_promtool() { - wget -q -O /tmp/prometheus.tar.gz https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}."${OS}"-amd64.tar.gz - tar zxfv /tmp/prometheus.tar.gz -C /tmp/ prometheus-${PROMETHEUS_VERSION}."${OS}"-amd64/promtool - ${SUDO} mv /tmp/prometheus-${PROMETHEUS_VERSION}."${OS}"-amd64/promtool /usr/local/bin/ - rmdir /tmp/prometheus-${PROMETHEUS_VERSION}."${OS}"-amd64 - rm /tmp/prometheus.tar.gz -} +[[ -n "${E2E_SETUP_KIND}" ]] && setup_kind -[[ -n "${E2E_SETUP_MINIKUBE}" ]] && setup_minikube - -minikube version +kind version [[ -n "${E2E_SETUP_KUBECTL}" ]] && setup_kubectl -export MINIKUBE_WANTUPDATENOTIFICATION=false -export MINIKUBE_WANTREPORTERRORPROMPT=false -export MINIKUBE_HOME=$HOME -export CHANGE_MINIKUBE_NONE_USER=true mkdir "${HOME}"/.kube || true touch "${HOME}"/.kube/config export KUBECONFIG=$HOME/.kube/config -${SUDO} minikube start --vm-driver="${MINIKUBE_DRIVER}" --kubernetes-version=${KUBERNETES_VERSION} --logtostderr -minikube update-context +kind create cluster --image="${NODE_IMAGE_NAME}:${KUBERNETES_VERSION}" + +kind export kubeconfig set +e is_kube_running="false" -# this for loop waits until kubectl can access the api server that Minikube has created +# this for loop waits until kubectl can access the api server that kind has created for _ in {1..90}; do # timeout for 3 minutes kubectl get po 1>/dev/null 2>&1 if [[ $? -ne 1 ]]; then @@ -93,12 +83,11 @@ for _ in {1..90}; do # timeout for 3 minutes break fi - echo "waiting for Kubernetes cluster up" + echo "waiting for Kubernetes cluster to come up" sleep 2 done if [[ ${is_kube_running} == "false" ]]; then - minikube logs echo "Kubernetes does not start within 3 minutes" exit 1 fi @@ -107,17 +96,16 @@ set -e kubectl version -# ensure that we build docker image in minikube -[[ "$MINIKUBE_DRIVER" != "none" ]] && eval "$(minikube docker-env)" - # query kube-state-metrics image tag -make container +REGISTRY="registry.k8s.io/kube-state-metrics" make container docker images -a -KUBE_STATE_METRICS_IMAGE_TAG=$(docker images -a|grep 'quay.io/coreos/kube-state-metrics'|grep -v 'latest'|awk '{print $2}'|sort -u) +KUBE_STATE_METRICS_IMAGE_TAG=$(docker images -a|grep "${KUBE_STATE_METRICS_IMAGE_NAME}" |grep -v 'latest'|awk '{print $2}'|sort -u) echo "local kube-state-metrics image tag: $KUBE_STATE_METRICS_IMAGE_TAG" +kind load docker-image "${KUBE_STATE_METRICS_IMAGE_NAME}:${KUBE_STATE_METRICS_IMAGE_TAG}" + # update kube-state-metrics image tag in deployment.yaml -sed -i.bak "s|${KUBE_STATE_METRICS_IMAGE_NAME}:v.*|${KUBE_STATE_METRICS_IMAGE_NAME}:${KUBE_STATE_METRICS_IMAGE_TAG}|g" ./examples/standard/deployment.yaml +sed -i.bak "s|${KUBE_STATE_METRICS_CURRENT_IMAGE_NAME}:v.*|${KUBE_STATE_METRICS_IMAGE_NAME}:${KUBE_STATE_METRICS_IMAGE_TAG}|g" ./examples/standard/deployment.yaml cat ./examples/standard/deployment.yaml trap finish EXIT @@ -150,7 +138,7 @@ for _ in {1..30}; do # timeout for 1 minutes break fi - echo "waiting for Kube-state-metrics up" + echo "waiting for kube-state-metrics to come up" sleep 2 done @@ -165,24 +153,17 @@ set -e echo "kube-state-metrics is up and running" echo "start e2e test for kube-state-metrics" -KSMURL='http://localhost:8001/api/v1/namespaces/kube-system/services/kube-state-metrics:http-metrics/proxy' -go test -v ./tests/e2e/ --ksmurl=${KSMURL} +KSM_HTTP_METRICS_URL='http://localhost:8001/api/v1/namespaces/kube-system/services/kube-state-metrics:http-metrics/proxy' +KSM_TELEMETRY_URL='http://localhost:8001/api/v1/namespaces/kube-system/services/kube-state-metrics:telemetry/proxy' +go test -v ./tests/e2e/main_test.go --ksm-http-metrics-url=${KSM_HTTP_METRICS_URL} --ksm-telemetry-url=${KSM_TELEMETRY_URL} +go test -v ./tests/e2e/hot-reload_test.go + +mkdir -p ${KUBE_STATE_METRICS_LOG_DIR} # TODO: re-implement the following test cases in Go with the goal of removing this file. echo "access kube-state-metrics metrics endpoint" curl -s "http://localhost:8001/api/v1/namespaces/kube-system/services/kube-state-metrics:http-metrics/proxy/metrics" >${KUBE_STATE_METRICS_LOG_DIR}/metrics -echo "check metrics format with promtool" -[[ -n "${E2E_SETUP_PROMTOOL}" ]] && setup_promtool -< ${KUBE_STATE_METRICS_LOG_DIR}/metrics promtool check metrics - -resources=$(find internal/store/ -maxdepth 1 -name "*.go" -not -name "*_test.go" -not -name "builder.go" -not -name "testutils.go" -not -name "utils.go" -print0 | xargs -0 -n1 basename | awk -F. '{print $1}'| grep -v "$EXCLUDED_RESOURCE_REGEX") -echo "available resources: $resources" -for resource in ${resources}; do - echo "checking that kube_${resource}* metrics exists" - grep "^kube_${resource}_" ${KUBE_STATE_METRICS_LOG_DIR}/metrics -done - KUBE_STATE_METRICS_STATUS=$(curl -s "http://localhost:8001/api/v1/namespaces/kube-system/services/kube-state-metrics:http-metrics/proxy/healthz") if [[ "${KUBE_STATE_METRICS_STATUS}" == "OK" ]]; then echo "kube-state-metrics is still running after accessing metrics endpoint" diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 0000000000..edebccf00b --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,12 @@ +To run these tests, you need to provide two CLI flags: + +- `--ksm-http-metrics-url`: url to access the kube-state-metrics service +- `--ksm-telemetry-url`: url to access the kube-state-metrics telemetry endpoint + +Example: + +``` +go test -v ./tests/e2e \ + --ksm-http-metrics-url=http://localhost:8080/ \ + --ksm-telemetry-url=http://localhost:8081/ +``` diff --git a/tests/e2e/framework.go b/tests/e2e/framework.go deleted file mode 100644 index 8eec05a0ae..0000000000 --- a/tests/e2e/framework.go +++ /dev/null @@ -1,125 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - "path" - "strings" -) - -const ( - epHealthz = "/healthz" - epMetrics = "/metrics" -) - -var ( - framework *Framework -) - -type Framework struct { - KsmClient *KSMClient -} - -func NewFramework(ksmurl string) (*Framework, error) { - ksmClient, err := NewKSMClient(ksmurl) - if err != nil { - return nil, err - } - - return &Framework{ - KsmClient: ksmClient, - }, nil -} - -type KSMClient struct { - endpoint *url.URL - client *http.Client -} - -func NewKSMClient(address string) (*KSMClient, error) { - u, err := url.Parse(address) - if err != nil { - return nil, err - } - u.Path = strings.TrimRight(u.Path, "/") - - return &KSMClient{ - endpoint: u, - client: &http.Client{}, - }, nil -} - -func (k *KSMClient) isHealthz() (bool, error) { - p := path.Join(k.endpoint.Path, epHealthz) - - u := *k.endpoint - u.Path = p - - req, err := http.NewRequest(http.MethodGet, u.String(), nil) - if err != nil { - return false, err - } - - resp, err := k.client.Do(req) - if err != nil { - return false, err - } - defer func() { - io.Copy(ioutil.Discard, resp.Body) - resp.Body.Close() - }() - - if resp.StatusCode != http.StatusOK { - return false, fmt.Errorf("server returned HTTP status %s", resp.Status) - } - - return true, nil -} - -func (k *KSMClient) metrics(w io.Writer) error { - p := path.Join(k.endpoint.Path, epMetrics) - - u := *k.endpoint - u.Path = p - - req, err := http.NewRequest(http.MethodGet, u.String(), nil) - if err != nil { - return err - } - - resp, err := k.client.Do(req) - if err != nil { - return err - } - defer func() { - io.Copy(ioutil.Discard, resp.Body) - resp.Body.Close() - }() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("server returned HTTP status %s", resp.Status) - } - - io.Copy(w, resp.Body) - - return nil -} diff --git a/tests/e2e/framework/framework.go b/tests/e2e/framework/framework.go new file mode 100644 index 0000000000..65e37463fa --- /dev/null +++ b/tests/e2e/framework/framework.go @@ -0,0 +1,173 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "bytes" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" + + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" +) + +const ( + epHealthz = "/healthz" + epMetrics = "/metrics" +) + +// The Framework stores a pointer to the KSMClient +type Framework struct { + KsmClient *KSMClient +} + +// New returns a new Framework given the kube-state-metrics service URLs. +// It delegates the url validation errs to NewKSMClient func. +func New(ksmHTTPMetricsURL, ksmTelemetryURL string) (*Framework, error) { + ksmClient, err := NewKSMClient(ksmHTTPMetricsURL, ksmTelemetryURL) + if err != nil { + return nil, err + } + + return &Framework{ + KsmClient: ksmClient, + }, nil +} + +// The KSMClient is the Kubernetes State Metric client. +type KSMClient struct { + httpMetricsEndpoint *url.URL + telemetryEndpoint *url.URL + client *http.Client +} + +// NewKSMClient retrieves a new KSMClient the kube-state-metrics service URLs. +// In case of error parsing the provided addresses, it returns an error. +func NewKSMClient(ksmHTTPMetricsAddress, ksmTelemetryAddress string) (*KSMClient, error) { + ksmHTTPMetricsURL, err := validateURL(ksmHTTPMetricsAddress) + if err != nil { + return nil, err + } + ksmTelemetryURL, err := validateURL(ksmTelemetryAddress) + if err != nil { + return nil, err + } + + return &KSMClient{ + httpMetricsEndpoint: ksmHTTPMetricsURL, + telemetryEndpoint: ksmTelemetryURL, + client: &http.Client{}, + }, nil +} + +func validateURL(address string) (*url.URL, error) { + u, err := url.Parse(address) + if err != nil { + return nil, err + } + u.Path = strings.TrimRight(u.Path, "/") + return u, nil +} + +// IsHealthz makes a request to the /healthz endpoint to get the health status. +func (k *KSMClient) IsHealthz() (bool, error) { + p := path.Join(k.httpMetricsEndpoint.Path, epHealthz) + + u := *k.httpMetricsEndpoint + u.Path = p + + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return false, err + } + + resp, err := k.client.Do(req) + if err != nil { + return false, err + } + defer func() { + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + return false, fmt.Errorf("server returned HTTP status %s", resp.Status) + } + + return true, nil +} + +func (k *KSMClient) writeMetrics(endpoint *url.URL, w io.Writer) error { + if endpoint == nil { + return errors.New("Endpoint is nil") + } + + u := *endpoint + u.Path = path.Join(endpoint.Path, epMetrics) + + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return err + } + + resp, err := k.client.Do(req) + if err != nil { + return err + } + defer func() { + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("server returned HTTP status %s", resp.Status) + } + + io.Copy(w, resp.Body) + + return nil +} + +// Metrics makes a request to the /metrics endpoint on the "http-metrics" port, +// and writes its content to the writer w. +func (k *KSMClient) Metrics(w io.Writer) error { + return k.writeMetrics(k.httpMetricsEndpoint, w) +} + +// TelemetryMetrics makes a request to the /metrics endpoint on the "telemetry" port, +// and writes its content to the writer w. +func (k *KSMClient) TelemetryMetrics(w io.Writer) error { + return k.writeMetrics(k.telemetryEndpoint, w) +} + +// ParseMetrics uses a prometheus TextParser to parse metrics, given a function +// that fetches and writes metrics. +func (f *Framework) ParseMetrics(metrics func(io.Writer) error) (map[string]*dto.MetricFamily, error) { + buf := &bytes.Buffer{} + err := metrics(buf) + if err != nil { + return nil, fmt.Errorf("Failed to get metrics: %w", err) + } + + parser := &expfmt.TextParser{} + return parser.TextToMetricFamilies(buf) +} diff --git a/tests/e2e/hot-reload_test.go b/tests/e2e/hot-reload_test.go new file mode 100644 index 0000000000..c5d099b58d --- /dev/null +++ b/tests/e2e/hot-reload_test.go @@ -0,0 +1,116 @@ +/* +Copyright 2022 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "net" + "os" + "testing" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + + "k8s.io/kube-state-metrics/v2/internal" + "k8s.io/kube-state-metrics/v2/pkg/options" +) + +func TestConfigHotReload(t *testing.T) { + + // Initialise options. + opts := options.NewOptions() + cmd := options.InitCommand + opts.AddFlags(cmd) + + // Create testdata. + f, err := os.CreateTemp("", "config") + if err != nil { + t.Fatal(err) + } + + // Delete artefacts. + defer func() { + err := os.Remove(opts.Config) + if err != nil { + t.Fatalf("failed to remove config file: %v", err) + } + }() + + // Populate options. + opts.Config = f.Name() + + // Assume $HOME is always defined. + opts.Kubeconfig = os.Getenv("HOME") + "/.kube/config" + + // Run general validation on options. + if err := opts.Parse(); err != nil { + t.Fatalf("failed to parse options: %v", err) + } + + // Make the process asynchronous. + go internal.RunKubeStateMetricsWrapper(opts) + + // Wait for port 8080 to come up. + err = wait.PollImmediate(1*time.Second, 20*time.Second, func() (bool, error) { + conn, err := net.Dial("tcp", "localhost:8080") + if err != nil { + return false, nil + } + err = conn.Close() + if err != nil { + return false, err + } + return true, nil + }) + if err != nil { + t.Fatalf("failed to wait for port 8080 to come up for the first time: %v", err) + } + + // Modify config to trigger hot reload. + config := `foo: "bar"` + err = os.WriteFile(opts.Config, []byte(config), 0600 /* rw------- */) + if err != nil { + t.Fatalf("failed to write config file: %v", err) + } + + // Wait for port 8080 to come up. + ch := make(chan bool, 1) + err = wait.PollImmediate(1*time.Second, 20*time.Second, func() (bool, error) { + conn, err := net.Dial("tcp", "localhost:8080") + if err != nil { + return false, nil + } + err = conn.Close() + if err != nil { + return false, err + } + return true, nil + }) + if err != nil { + t.Fatalf("failed to wait for port 8080 to come up after restarting the process: %v", err) + } + + // Indicate that the test has passed. + ch <- true + + // Wait for process to exit. + select { + case <-ch: + t.Log("test passed successfully") + case <-time.After(20 * time.Second): + t.Fatal("timed out waiting for test to pass, check the logs for more info") + } +} diff --git a/tests/e2e/main_test.go b/tests/e2e/main_test.go index eb6085b3c6..560fc071c0 100644 --- a/tests/e2e/main_test.go +++ b/tests/e2e/main_test.go @@ -17,21 +17,38 @@ limitations under the License. package e2e import ( + "bufio" "bytes" "flag" + "fmt" "log" "os" + "path" + "path/filepath" + "regexp" + "sort" + "strings" "testing" - "github.com/prometheus/prometheus/util/promlint" + "github.com/prometheus/client_golang/prometheus/testutil/promlint" + dto "github.com/prometheus/client_model/go" + + ksmFramework "k8s.io/kube-state-metrics/v2/tests/e2e/framework" ) +var framework *ksmFramework.Framework + func TestMain(m *testing.M) { - ksmurl := flag.String( - "ksmurl", + ksmHTTPMetricsURL := flag.String( + "ksm-http-metrics-url", "", "url to access the kube-state-metrics service", ) + ksmTelemetryURL := flag.String( + "ksm-telemetry-url", + "", + "url to access the kube-state-metrics telemetry endpoint", + ) flag.Parse() var ( @@ -39,7 +56,7 @@ func TestMain(m *testing.M) { exitCode int ) - if framework, err = NewFramework(*ksmurl); err != nil { + if framework, err = ksmFramework.New(*ksmHTTPMetricsURL, *ksmTelemetryURL); err != nil { log.Fatalf("failed to setup framework: %v\n", err) } @@ -49,7 +66,7 @@ func TestMain(m *testing.M) { } func TestIsHealthz(t *testing.T) { - ok, err := framework.KsmClient.isHealthz() + ok, err := framework.KsmClient.IsHealthz() if err != nil { t.Fatalf("kube-state-metrics healthz check failed: %v", err) } @@ -62,7 +79,7 @@ func TestIsHealthz(t *testing.T) { func TestLintMetrics(t *testing.T) { buf := &bytes.Buffer{} - err := framework.KsmClient.metrics(buf) + err := framework.KsmClient.Metrics(buf) if err != nil { t.Fatalf("failed to get metrics from kube-state-metrics: %v", err) } @@ -77,3 +94,224 @@ func TestLintMetrics(t *testing.T) { t.Fatalf("the problems encountered in Lint are: %v", problems) } } + +func TestDocumentation(t *testing.T) { + labelsDocumentation, err := getLabelsDocumentation() + if err != nil { + t.Fatal("Cannot get labels documentation", err) + } + + metricFamilies, err := framework.ParseMetrics(framework.KsmClient.Metrics) + if err != nil { + t.Fatal("Failed to get or decode metrics", err) + } + + for _, metricFamily := range metricFamilies { + metric := metricFamily.GetName() + + acceptedLabelNames, ok := labelsDocumentation[metric] + if !ok { + t.Errorf("Metric %s not found in documentation.", metric) + continue + } + for _, m := range metricFamily.Metric { + for _, l := range m.Label { + labelName := l.GetName() + labelNameMatched := false + for _, labelPattern := range acceptedLabelNames { + re, err := regexp.Compile(labelPattern) + if err != nil { + t.Errorf("Cannot compile pattern %s: %v", labelPattern, err) + continue + } + if re.MatchString(labelName) { + labelNameMatched = true + break + } + } + if !labelNameMatched { + t.Errorf("Label %s not found in documentation. Documented labels for metric %s are: %s", + labelName, metric, strings.Join(acceptedLabelNames, ", ")) + } + } + } + } +} + +// getLabelsDocumentation is a helper function that gets metric mabels documentation. +// It returns a map where keys are metric names, and values are slices of label names, +// and an error in case of failure. +// By convention, UPPER_CASE parts in label names denotes wilcard patterns, used for dynamic labels. +func getLabelsDocumentation() (map[string][]string, error) { + documentedMetrics := map[string][]string{} + + docPath := "../../docs/" + docFiles, err := os.ReadDir(docPath) + if err != nil { + return nil, fmt.Errorf("failed to read documentation directory: %w", err) + } + + // Match file names such as daemonset-metrics.md + fileRe := regexp.MustCompile(`^([a-z]*)-metrics.md$`) + // Match doc lines such as | kube_node_created | Gauge | `node`=<node-address>| STABLE | + lineRe := regexp.MustCompile(`^\| *(kube_[a-z_]+) *\| *[a-zA-Z]+ *\|(.*)\| *[A-Z]+`) + // Match label names in label documentation + labelsRe := regexp.MustCompile("`([a-zA-Z_][a-zA-Z0-9_]*)`") + // Match wildcard patterns for dynamic labels such as label_CRONJOB_LABEL + patternRe := regexp.MustCompile(`_[A-Z_]+`) + + for _, file := range docFiles { + if file.IsDir() || !fileRe.MatchString(file.Name()) { + continue + } + + filePath := path.Join(docPath, file.Name()) + f, err := os.Open(filepath.Clean(filePath)) + if err != nil { + return nil, fmt.Errorf("cannot read file %s: %w", filePath, err) + } + scanner := bufio.NewScanner(f) + for scanner.Scan() { + params := lineRe.FindStringSubmatch(scanner.Text()) + if len(params) != 3 { + continue + } + metric := params[1] + labelsDoc := params[2] + + labels := labelsRe.FindAllStringSubmatch(labelsDoc, -1) + labelPatterns := make([]string, len(labels)) + for i, l := range labels { + if len(l) <= 1 { + return nil, fmt.Errorf("Label documentation %s did not match regex", labelsDoc) + } + labelPatterns[i] = patternRe.ReplaceAllString(l[1], "_.*") + } + + documentedMetrics[metric] = labelPatterns + } + } + return documentedMetrics, nil +} + +func TestKubeStateMetricsErrorMetrics(t *testing.T) { + metricFamilies, err := framework.ParseMetrics(framework.KsmClient.TelemetryMetrics) + if err != nil { + t.Fatal("Failed to get or decode telemetry metrics", err) + } + + // This map's keys are the metrics expected in kube-state-metrics telemetry. + // Its values are booleans, set to true when the metric is found. + foundMetricFamily := map[string]bool{ + "kube_state_metrics_list_total": false, + "kube_state_metrics_watch_total": false, + } + + for _, metricFamily := range metricFamilies { + name := metricFamily.GetName() + if _, expectedMetric := foundMetricFamily[name]; expectedMetric { + foundMetricFamily[name] = true + + for _, m := range metricFamily.Metric { + if hasLabelError(m) && m.GetCounter().GetValue() > 0 { + t.Errorf("Metric %s in telemetry shows a list/watch error", prettyPrintCounter(name, m)) + } + } + } + } + + for metricFamily, found := range foundMetricFamily { + if !found { + t.Errorf("Metric family %s was not found in telemetry metrics", metricFamily) + } + } +} + +func hasLabelError(metric *dto.Metric) bool { + for _, l := range metric.Label { + if l.GetName() == "result" && l.GetValue() == "error" { + return true + } + } + return false +} + +func prettyPrintCounter(name string, metric *dto.Metric) string { + labelStrings := []string{} + for _, l := range metric.Label { + labelStrings = append(labelStrings, fmt.Sprintf(`%s="%s"`, l.GetName(), l.GetValue())) + } + return fmt.Sprintf("%s{%s} %d", name, strings.Join(labelStrings, ","), int(metric.GetCounter().GetValue())) +} + +func TestDefaultCollectorMetricsAvailable(t *testing.T) { + buf := &bytes.Buffer{} + + err := framework.KsmClient.Metrics(buf) + if err != nil { + t.Fatalf("failed to get metrics from kube-state-metrics: %v", err) + } + + resources := map[string]struct{}{} + nonDefaultResources := map[string]bool{ + "clusterrole": true, + "clusterrolebinding": true, + "endpointslice": true, + "ingressclass": true, + "role": true, + "rolebinding": true, + "serviceaccount": true, + "verticalpodautoscaler": true, + } + nonResources := map[string]bool{ + "builder": true, + "utils": true, + "testutils": true, + } + + files, err := os.ReadDir("../../internal/store/") + if err != nil { + t.Fatalf("failed to read dir to get all resources name: %v", err) + } + + re := regexp.MustCompile(`^([a-z]+).go$`) + for _, file := range files { + params := re.FindStringSubmatch(file.Name()) + if len(params) != 2 { + continue + } + if nonResources[params[1]] { + // Non resource file + continue + } + if nonDefaultResources[params[1]] { + // Resource disabled by default + continue + } + resources[params[1]] = struct{}{} + } + + re = regexp.MustCompile(`^kube_([a-z]+)_`) + scanner := bufio.NewScanner(buf) + for scanner.Scan() { + params := re.FindStringSubmatch(scanner.Text()) + if len(params) != 2 { + continue + } + delete(resources, params[1]) + } + + err = scanner.Err() + if err != nil { + t.Fatalf("failed to scan metrics: %v", err) + } + + if len(resources) != 0 { + s := []string{} + for k := range resources { + s = append(s, k) + } + sort.Strings(s) + t.Fatalf("failed to find metrics of resources: %s", strings.Join(s, ", ")) + } +} diff --git a/tests/lib/lib_test.go b/tests/lib/lib_test.go index ebc9343df0..ba96edf00e 100644 --- a/tests/lib/lib_test.go +++ b/tests/lib/lib_test.go @@ -30,8 +30,8 @@ import ( "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/tools/cache" - "k8s.io/kube-state-metrics/pkg/metric" - metricsstore "k8s.io/kube-state-metrics/pkg/metrics_store" + "k8s.io/kube-state-metrics/v2/pkg/metric" + metricsstore "k8s.io/kube-state-metrics/v2/pkg/metrics_store" ) func TestAsLibrary(t *testing.T) { @@ -44,7 +44,7 @@ func TestAsLibrary(t *testing.T) { }, } - _, err := kubeClient.CoreV1().Services(metav1.NamespaceDefault).Create(&service) + _, err := kubeClient.CoreV1().Services(metav1.NamespaceDefault).Create(context.TODO(), &service, metav1.CreateOptions{}) if err != nil { t.Fatal(err) } @@ -55,7 +55,11 @@ func TestAsLibrary(t *testing.T) { time.Sleep(time.Second) w := strings.Builder{} - c.WriteAll(&w) + mw := metricsstore.NewMetricsWriter(c) + err = mw.WriteAll(&w) + if err != nil { + t.Fatalf("failed to write metrics: %v", err) + } m := w.String() if !strings.Contains(m, service.ObjectMeta.Name) { @@ -68,10 +72,10 @@ func serviceCollector(kubeClient clientset.Interface) *metricsstore.MetricsStore lw := cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - return kubeClient.CoreV1().Services(metav1.NamespaceDefault).List(opts) + return kubeClient.CoreV1().Services(metav1.NamespaceDefault).List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - return kubeClient.CoreV1().Services(metav1.NamespaceDefault).Watch(opts) + return kubeClient.CoreV1().Services(metav1.NamespaceDefault).Watch(context.TODO(), opts) }, } @@ -82,7 +86,7 @@ func serviceCollector(kubeClient clientset.Interface) *metricsstore.MetricsStore return store } -func generateServiceMetrics(obj interface{}) []metricsstore.FamilyByteSlicer { +func generateServiceMetrics(obj interface{}) []metric.FamilyInterface { sPointer := obj.(*v1.Service) s := *sPointer @@ -97,5 +101,5 @@ func generateServiceMetrics(obj interface{}) []metricsstore.FamilyByteSlicer { Metrics: []*metric.Metric{&m}, } - return []metricsstore.FamilyByteSlicer{&family} + return []metric.FamilyInterface{&family} } diff --git a/tests/manifests/clusterole.yaml b/tests/manifests/clusterole.yaml new file mode 100644 index 0000000000..c7c6fea1fa --- /dev/null +++ b/tests/manifests/clusterole.yaml @@ -0,0 +1,8 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: clusterrole +rules: +- apiGroups: [""] + resources: [""] + verbs: ["get", "watch", "list"] \ No newline at end of file diff --git a/tests/manifests/clusterrolebinding.yaml b/tests/manifests/clusterrolebinding.yaml new file mode 100644 index 0000000000..fc8c3a58d9 --- /dev/null +++ b/tests/manifests/clusterrolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: clusterrolebinding +subjects: +- kind: ServiceAccount + name: test-service-account + namespace: default +roleRef: + kind: ClusterRole + name: clusterrole + apiGroup: rbac.authorization.k8s.io diff --git a/tests/manifests/cronjob.yaml b/tests/manifests/cronjob.yaml index d08da4f458..118f238380 100644 --- a/tests/manifests/cronjob.yaml +++ b/tests/manifests/cronjob.yaml @@ -1,4 +1,4 @@ -apiVersion: batch/v1beta1 +apiVersion: batch/v1 kind: CronJob metadata: name: cronjob diff --git a/tests/manifests/csr.yaml b/tests/manifests/csr.yaml index 9f6a718633..f09ae6d801 100644 --- a/tests/manifests/csr.yaml +++ b/tests/manifests/csr.yaml @@ -1,4 +1,4 @@ -apiVersion: certificates.k8s.io/v1beta1 +apiVersion: certificates.k8s.io/v1 kind: CertificateSigningRequest metadata: name: my-svc.my-namespace @@ -7,6 +7,7 @@ spec: - system:masters - system:authenticated request: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURSBSRVFVRVNULS0tLS0KTUlJQllqQ0NBUWdDQVFBd01ERXVNQ3dHQTFVRUF4TWxiWGt0Y0c5a0xtMTVMVzVoYldWemNHRmpaUzV3YjJRdQpZMngxYzNSbGNpNXNiMk5oYkRCWk1CTUdCeXFHU000OUFnRUdDQ3FHU000OUF3RUhBMElBQkpHai9CazNkTVNjCmNmQWdqbWk2d2wxdVYrQVNIR1g1ZHluWHFWdmJsaUd4clFBL2FFOWY0NDc5eFpVR0lDNjFPSmwrR0JJVGhBV0cKWlFiTEhDQ0xscXVnZGpCMEJna3Foa2lHOXcwQkNRNHhaekJsTUdNR0ExVWRFUVJjTUZxQ0pXMTVMWE4yWXk1dAplUzF1WVcxbGMzQmhZMlV1YzNaakxtTnNkWE4wWlhJdWJHOWpZV3lDSlcxNUxYQnZaQzV0ZVMxdVlXMWxjM0JoClkyVXVjRzlrTG1Oc2RYTjBaWEl1Ykc5allXeUhCS3lvQUJpSEJBb0FJZ0l3Q2dZSUtvWkl6ajBFQXdJRFNBQXcKUlFJZ1psb0J6Vkp4UkpjeUlweHZ1WGhTWFRhM3lPaXJDVVRCZytqQk5DUUcyT29DSVFDQVV6c2IzYWxuV1ljdAp5eGxEVEgxZkF6dms3R0ZINVVhd3RwaitWREFJNHc9PQotLS0tLUVORCBDRVJUSUZJQ0FURSBSRVFVRVNULS0tLS0K + signerName: kubernetes.io/kube-apiserver-client usages: - digital signature - key encipherment diff --git a/tests/manifests/endpointslice.yaml b/tests/manifests/endpointslice.yaml new file mode 100644 index 0000000000..078e67f2bb --- /dev/null +++ b/tests/manifests/endpointslice.yaml @@ -0,0 +1,13 @@ +apiVersion: discovery.k8s.io/v1 +kind: EndpointSlice +metadata: + name: example-endpointslice + namespace: default +addressType: IPv4 +endpoints: +- addresses: + - 10.0.0.2 + conditions: + ready: true + serving: true + terminating: false diff --git a/tests/manifests/ingress.yaml b/tests/manifests/ingress.yaml index d0fd3e6b9f..7e35f7e343 100644 --- a/tests/manifests/ingress.yaml +++ b/tests/manifests/ingress.yaml @@ -1,4 +1,4 @@ -apiVersion: extensions/v1beta1 +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: example-ingress @@ -9,10 +9,16 @@ spec: - http: paths: - path: /apple + pathType: Prefix backend: - serviceName: apple-service - servicePort: 5678 + service: + name: apple-service + port: + number: 5678 - path: /banana + pathType: Prefix backend: - serviceName: banana-service - servicePort: 5678 + service: + name: banana-service + port: + number: 5678 diff --git a/tests/manifests/ingressclass.yaml b/tests/manifests/ingressclass.yaml new file mode 100644 index 0000000000..95a5aa1278 --- /dev/null +++ b/tests/manifests/ingressclass.yaml @@ -0,0 +1,8 @@ +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: example-ingressclass + annotations: + ingressclass.kubernetes.io/is-default-class: "true" +spec: + controller: example-ingress/controller diff --git a/tests/manifests/poddisruptionbudget.yaml b/tests/manifests/poddisruptionbudget.yaml index cb980b7b83..19c2b9222f 100644 --- a/tests/manifests/poddisruptionbudget.yaml +++ b/tests/manifests/poddisruptionbudget.yaml @@ -1,4 +1,4 @@ -apiVersion: policy/v1beta1 +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: pdb diff --git a/tests/manifests/role.yaml b/tests/manifests/role.yaml new file mode 100644 index 0000000000..266e4e656a --- /dev/null +++ b/tests/manifests/role.yaml @@ -0,0 +1,8 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: role +rules: +- apiGroups: [""] + resources: [""] + verbs: ["get", "watch", "list"] \ No newline at end of file diff --git a/tests/manifests/rolebinding.yaml b/tests/manifests/rolebinding.yaml new file mode 100644 index 0000000000..7b01c65bea --- /dev/null +++ b/tests/manifests/rolebinding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rolebinding +subjects: +- kind: ServiceAccount + name: test-service-account +roleRef: + kind: Role + name: role + apiGroup: rbac.authorization.k8s.io diff --git a/tests/manifests/serviceaccount.yaml b/tests/manifests/serviceaccount.yaml new file mode 100644 index 0000000000..558a5f5e24 --- /dev/null +++ b/tests/manifests/serviceaccount.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-service-account diff --git a/tests/manifests/volumeattachment.yaml b/tests/manifests/volumeattachment.yaml index 9ed81a8cd8..8bc880609e 100644 --- a/tests/manifests/volumeattachment.yaml +++ b/tests/manifests/volumeattachment.yaml @@ -4,6 +4,6 @@ metadata: name: volumeattachment spec: attacher: attacher - nodeName: minikube + nodeName: kind source: persistentVolumeName: persistentvolume diff --git a/tests/rules/alerts-test.yaml b/tests/rules/alerts-test.yaml new file mode 100644 index 0000000000..12987cfa47 --- /dev/null +++ b/tests/rules/alerts-test.yaml @@ -0,0 +1,88 @@ +rule_files: + - ../../examples/prometheus-alerting-rules/alerts.yaml + +evaluation_interval: 1m + +tests: + # Test long-lasting invalid configuration should trigger alert + - interval: 1m + input_series: + # T=0: 3 shards with correct configuration + # T=9m: shard #0 was incorrectly redeployed with --total-shards=1 + - series: 'kube_state_metrics_total_shards{job="kube-state-metrics",pod="kube-state-metrics-shard-0-0"}' + values: '3x9 1x21' + - series: 'kube_state_metrics_total_shards{job="kube-state-metrics",pod="kube-state-metrics-shard-1-0"}' + values: '3x30' + - series: 'kube_state_metrics_total_shards{job="kube-state-metrics",pod="kube-state-metrics-shard-2-0"}' + values: '3x30' + alert_rule_test: + - eval_time: 25m + alertname: KubeStateMetricsShardingMismatch + exp_alerts: + - exp_labels: + severity: critical + exp_annotations: + summary: "kube-state-metrics sharding is misconfigured." + description: "kube-state-metrics pods are running with different --total-shards configuration, some Kubernetes objects may be exposed multiple times or not exposed at all." + + # Test deployment from one shard to multiple shards should not trigger alert + - interval: 1m + input_series: + # T=0: 1 shard with correct configuration + # T=9m: two more shards were deployed, all shards were correctly configured + - series: 'kube_state_metrics_total_shards{job="kube-state-metrics",pod="kube-state-metrics-shard-0-0"}' + values: '1x9 3x21' + - series: 'kube_state_metrics_total_shards{job="kube-state-metrics",pod="kube-state-metrics-shard-1-0"}' + values: '_x9 3x21' + - series: 'kube_state_metrics_total_shards{job="kube-state-metrics",pod="kube-state-metrics-shard-2-0"}' + values: '_x9 3x21' + - series: 'kube_state_metrics_shard_ordinal{job="kube-state-metrics",pod="kube-state-metrics-shard-0-0",shard_ordinal="0"}' + values: '0x30' + - series: 'kube_state_metrics_shard_ordinal{job="kube-state-metrics",pod="kube-state-metrics-shard-1-0",shard_ordinal="1"}' + values: '_x9 1x21' + - series: 'kube_state_metrics_shard_ordinal{job="kube-state-metrics",pod="kube-state-metrics-shard-2-0",shard_ordinal="2"}' + values: '_x9 2x21' + alert_rule_test: [] + + # Test misconfigured single shard deployment + - interval: 1m + input_series: + - series: 'kube_state_metrics_total_shards{job="kube-state-metrics",pod="kube-state-metrics-shard-0-0"}' + values: '3x30' + - series: 'kube_state_metrics_shard_ordinal{job="kube-state-metrics",pod="kube-state-metrics-shard-0-0",shard_ordinal="0"}' + values: '0x30' + alert_rule_test: + - eval_time: 25m + alertname: KubeStateMetricsShardsMissing + exp_alerts: + - exp_labels: + severity: critical + exp_annotations: + summary: "kube-state-metrics shards are missing." + description: "kube-state-metrics shards are missing, some Kubernetes objects are not being exposed." + + # Test misconfigured single shard deployment + - interval: 1m + input_series: + - series: 'kube_state_metrics_total_shards{job="kube-state-metrics",pod="kube-state-metrics-shard-0-0"}' + values: '3x30' + - series: 'kube_state_metrics_shard_ordinal{job="kube-state-metrics",pod="kube-state-metrics-shard-0-0",shard_ordinal="0"}' + values: '0x30' + alert_rule_test: + - eval_time: 25m + alertname: KubeStateMetricsShardsMissing + exp_alerts: + - exp_labels: + severity: critical + exp_annotations: + summary: "kube-state-metrics shards are missing." + description: "kube-state-metrics shards are missing, some Kubernetes objects are not being exposed." + promql_expr_test: + # Test that missing shards are #2 (0b100) and #1 (0b010) + - expr: | + 2^max(kube_state_metrics_total_shards{job="kube-state-metrics"}) - 1 + - + sum( 2 ^ max by (shard_ordinal) (kube_state_metrics_shard_ordinal{job="kube-state-metrics"}) ) + eval_time: 25m + exp_samples: + - value: 0b110 diff --git a/tools/go.mod b/tools/go.mod new file mode 100644 index 0000000000..9a11c4c830 --- /dev/null +++ b/tools/go.mod @@ -0,0 +1,27 @@ +module k8s.io/kube-state-metrics/v2/tools + +go 1.19 + +require ( + github.com/brancz/gojsontoyaml v0.1.0 + github.com/campoy/embedmd v1.0.0 + github.com/google/go-jsonnet v0.19.1 + github.com/jsonnet-bundler/jsonnet-bundler v0.5.1 + golang.org/x/perf v0.0.0-20230113213139-801c7ef9e5c5 +) + +require ( + github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 // indirect + github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect + github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect + github.com/fatih/color v1.13.0 // indirect + github.com/ghodss/yaml v1.0.0 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/sys v0.1.0 // indirect + gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + sigs.k8s.io/yaml v1.1.0 // indirect +) diff --git a/tools/go.sum b/tools/go.sum new file mode 100644 index 0000000000..cf3a64f682 --- /dev/null +++ b/tools/go.sum @@ -0,0 +1,171 @@ +cloud.google.com/go v0.0.0-20170206221025-ce650573d812/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20190129172621-c8b1d7a94ddf/go.mod h1:aJ4qN3TfrelA6NZ6AXsXRfmEVaYin3EDbSPJrKS8OXo= +github.com/aclements/go-gg v0.0.0-20170118225347-6dbb4e4fefb0/go.mod h1:55qNq4vcpkIuHowELi5C8e+1yUHtoLoOUR9QU5j7Tes= +github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 h1:xlwdaKcTNVW4PtpQb8aKA4Pjy0CdJHEqvFbAnvR5m2g= +github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794/go.mod h1:7e+I0LQFUI9AXWxOfsQROs9xPhoJtbsyWcjJqDd4KPY= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/ajstarks/svgo v0.0.0-20210923152817-c3b6e2f0c527/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/brancz/gojsontoyaml v0.1.0 h1:SdzR3+BCVOqaI42nFGTeaB7/2DgDM4fhuvRLqxatA8M= +github.com/brancz/gojsontoyaml v0.1.0/go.mod h1:+ycZY94+V11XZBUaDEsbLr3hPNS/ZPrDVKKNUg3Sgvg= +github.com/campoy/embedmd v1.0.0 h1:V4kI2qTJJLf4J29RzI/MAt2c3Bl4dQSYPuflzwFH2hY= +github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= +github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= +github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= +github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= +github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac/go.mod h1:P32wAyui1PQ58Oce/KYkOqQv8cVw1zAapXOl+dRFGbc= +github.com/gonum/floats v0.0.0-20181209220543-c233463c7e82/go.mod h1:PxC8OnwL11+aosOB5+iEPoV3picfs8tUpkVd0pDo+Kg= +github.com/gonum/internal v0.0.0-20181124074243-f884aa714029/go.mod h1:Pu4dmpkhSyOzRwuXkOgAvijx4o+4YMUJJo9OvPYMkks= +github.com/gonum/lapack v0.0.0-20181123203213-e4cdc5a0bff9/go.mod h1:XA3DeT6rxh2EAE789SSiSJNqxPaC0aE9J8NTOI0Jo/A= +github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9/go.mod h1:0EXg4mc1CNP0HCqCz+K4ts155PXIlUywf0wqN+GfPZw= +github.com/google/go-jsonnet v0.19.1 h1:MORxkrG0elylUqh36R4AcSPX0oZQa9hvI3lroN+kDhs= +github.com/google/go-jsonnet v0.19.1/go.mod h1:5JVT33JVCoehdTj5Z2KJq1eIdt3Nb8PCmZ+W5D8U350= +github.com/google/safehtml v0.0.2/go.mod h1:L4KWwDsUJdECRAEpZoBn3O64bQaywRscowZjJAzjHnU= +github.com/googleapis/gax-go v0.0.0-20161107002406-da06d194a00e/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/jsonnet-bundler/jsonnet-bundler v0.5.1 h1:eUd6EA1Qzz73Q4NLNLOrNkMb96+6NTTERbX9lqaxVwk= +github.com/jsonnet-bundler/jsonnet-bundler v0.5.1/go.mod h1:Qrdw/7mOFS2SKCOALKFfEH8gdvXJi8XZjw9g5ilpf4I= +github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= +github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= +github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= +github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.4 h1:wZRexSlwd7ZXfKINDLsO4r7WBt3gTKONc6K/VesHvHM= +github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20170207211851-4464e7848382/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/perf v0.0.0-20230113213139-801c7ef9e5c5 h1:ObuXPmIgI4ZMyQLIz48cJYgSyWdjUXc2SZAdyJMwEAU= +golang.org/x/perf v0.0.0-20230113213139-801c7ef9e5c5/go.mod h1:UBKtEnL8aqnd+0JHqZ+2qoMDwtuy6cYhhKNoHLBiTQc= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= +gonum.org/v1/plot v0.10.0/go.mod h1:JWIHJ7U20drSQb/aDpTetJzfC1KlAPldJLpkSy88dvQ= +google.golang.org/api v0.0.0-20170206182103-3d017632ea10/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/grpc v0.0.0-20170208002647-2a6bf6142e96/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/tools/tools.go b/tools/tools.go index 50667898b9..36ac07700c 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -1,3 +1,4 @@ +//go:build tools // +build tools /* @@ -23,5 +24,5 @@ import ( _ "github.com/campoy/embedmd" _ "github.com/google/go-jsonnet/cmd/jsonnet" _ "github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb" - _ "golang.org/x/tools/cmd/benchcmp" + _ "golang.org/x/perf/cmd/benchstat" )