From 8c5712e404268946a0558f7d79abbf89e50fda36 Mon Sep 17 00:00:00 2001 From: rambohe-ch Date: Sat, 30 Mar 2024 12:07:44 +0800 Subject: [PATCH] 1. upgrade k8s.io/xxx to v0.28.9 2. upgrade controller-runtime to v0.16.5 --- .github/workflows/ci.yaml | 15 +- .gitignore | 1 + Makefile | 8 +- .../crds/iot.openyurt.io_platformadmins.yaml | 2583 +++++++++++------ .../network.openyurt.io_poolservices.yaml | 15 +- cmd/yurt-iot-dock/app/core.go | 10 +- cmd/yurt-iot-dock/yurt-iot-dock.go | 3 - cmd/yurt-manager/app/manager.go | 31 +- cmd/yurt-tunnel-agent/app/start.go | 5 +- cmd/yurt-tunnel-server/app/config/config.go | 2 +- cmd/yurt-tunnel-server/app/options/options.go | 4 +- cmd/yurt-tunnel-server/app/start.go | 8 +- cmd/yurthub/app/config/config.go | 3 +- cmd/yurthub/app/start.go | 4 +- go.mod | 240 +- go.sum | 1886 ++++++++---- .../release/Dockerfile.node-servant | 2 +- .../release/Dockerfile.yurt-iot-dock | 2 +- .../release/Dockerfile.yurt-manager | 2 +- .../release/Dockerfile.yurt-tunnel-agent | 2 +- .../release/Dockerfile.yurt-tunnel-server | 2 +- hack/dockerfiles/release/Dockerfile.yurthub | 2 +- hack/make-rules/image_build.sh | 5 +- hack/make-rules/local-up-openyurt.sh | 4 +- hack/make-rules/run-e2e-tests.sh | 37 +- pkg/apis/apps/v1alpha1/default.go | 20 +- pkg/node-servant/components/yurthub.go | 5 +- .../static-pod-upgrade/util/pods.go | 5 +- pkg/util/iptables/iptables.go | 251 +- pkg/util/iptables/iptables_linux.go | 17 +- pkg/util/iptables/iptables_test.go | 334 ++- pkg/util/iptables/iptables_unsupported.go | 7 +- .../iptables/iptables_unsupported_test.go | 35 - pkg/util/iptables/monitor_test.go | 338 --- pkg/util/iptables/save_restore.go | 104 +- pkg/util/iptables/save_restore_test.go | 97 +- pkg/util/iptables/testing/fake.go | 323 ++- pkg/util/iptables/testing/fake_test.go | 908 ++---- pkg/util/iptables/testing/parse.go | 375 +++ pkg/util/iptables/testing/parse_test.go | 595 ++++ .../kubeadm/app/util/apiclient/idempotency.go | 4 +- pkg/util/taints/taints.go | 10 +- pkg/yurtadm/cmd/join/join.go | 6 +- pkg/yurtadm/cmd/join/join_test.go | 6 +- pkg/yurtadm/cmd/join/joindata/data.go | 2 +- pkg/yurtadm/util/kubernetes/kubernetes.go | 2 +- pkg/yurtadm/util/yurthub/yurthub.go | 5 +- pkg/yurtadm/util/yurthub/yurthub_test.go | 2 +- pkg/yurthub/cachemanager/cache_agent.go | 14 +- pkg/yurthub/cachemanager/cache_agent_test.go | 26 +- pkg/yurthub/cachemanager/cache_manager.go | 33 - .../certificate/manager/manager_test.go | 5 +- pkg/yurthub/certificate/server/server.go | 2 +- pkg/yurthub/filter/approver/approver.go | 30 +- pkg/yurthub/filter/approver/approver_test.go | 65 +- pkg/yurthub/filter/base/base.go | 12 +- pkg/yurthub/filter/base/base_test.go | 22 +- .../filter/discardcloudservice/filter.go | 6 +- .../filter/discardcloudservice/filter_test.go | 2 +- .../filter/forwardkubesvctraffic/filter.go | 6 +- .../forwardkubesvctraffic/filter_test.go | 2 +- pkg/yurthub/filter/inclusterconfig/filter.go | 6 +- .../filter/inclusterconfig/filter_test.go | 2 +- .../filter/initializer/initializer_test.go | 4 +- .../initializer/node_initializer_test.go | 14 +- pkg/yurthub/filter/interfaces.go | 2 +- pkg/yurthub/filter/manager/manager.go | 2 +- pkg/yurthub/filter/manager/manager_test.go | 14 +- pkg/yurthub/filter/masterservice/filter.go | 6 +- .../filter/masterservice/filter_test.go | 2 +- .../filter/nodeportisolation/filter.go | 12 +- .../filter/nodeportisolation/filter_test.go | 2 +- pkg/yurthub/filter/responsefilter/filter.go | 4 +- .../filter/responsefilter/filter_test.go | 16 +- pkg/yurthub/filter/servicetopology/filter.go | 8 +- .../filter/servicetopology/filter_test.go | 2 +- pkg/yurthub/healthchecker/node_lease.go | 4 +- pkg/yurthub/healthchecker/node_lease_test.go | 2 +- pkg/yurthub/kubernetes/rest/config_test.go | 3 +- pkg/yurthub/network/iptables.go | 4 +- pkg/yurthub/otaupdate/ota.go | 2 +- pkg/yurthub/otaupdate/ota_test.go | 8 +- pkg/yurthub/otaupdate/upgrader/daemon_pod.go | 4 +- pkg/yurthub/proxy/util/util.go | 2 +- pkg/yurthub/server/certificate_test.go | 5 +- pkg/yurthub/server/nonresource.go | 2 +- pkg/yurthub/server/nonresource_test.go | 6 +- pkg/yurthub/server/server.go | 2 +- pkg/yurthub/util/util.go | 2 +- .../certmanager/certmanager_test.go | 17 +- .../yurtcoordinator/coordinator_test.go | 12 +- pkg/yurthub/yurtcoordinator/informer_lease.go | 4 +- pkg/yurtiotdock/controllers/util/tools.go | 6 +- .../csrapprover/csrapprover_controller.go | 48 +- .../daemon_pod_updater_controller.go | 26 +- .../daemon_pod_updater_controller_test.go | 2 +- .../kubernetes/controller_utils_test.go | 6 +- .../controller/daemonpodupdater/util.go | 2 +- .../internal/controller/controller.go | 44 +- .../loadbalancerset/event_handler.go | 16 +- .../loadbalancerset/event_handler_test.go | 43 +- .../loadbalancerset_controller.go | 19 +- .../loadbalancerset_controller_test.go | 18 +- .../loadbalancerset/map_utils.go | 10 +- .../nodebucket/nodebucket_controller.go | 39 +- .../nodebucket/nodebucket_controller_test.go | 20 +- .../node_lifecycle_controller.go | 14 +- .../node_lifecycle_controller_test.go | 12 +- .../scheduler/rate_limited_queue.go | 6 +- .../scheduler/rate_limited_queue_test.go | 18 +- .../scheduler/taint_manager_test.go | 6 +- .../nodepool/nodepool_controller.go | 19 +- .../nodepool/nodepool_controller_test.go | 2 +- .../nodepool/nodepool_enqueue_handlers.go | 9 +- .../nodepool_enqueue_handlers_test.go | 7 +- .../controller/platformadmin/config/config.go | 8 +- .../controller/platformadmin/iotdock.go | 4 +- .../platformadmin/platformadmin_controller.go | 26 +- .../controller/raven/dns/dns_controller.go | 6 +- .../raven/dns/dns_enqueue_handlers.go | 18 +- .../raven/dns/dns_enqueue_handlers_test.go | 11 +- .../gateway_internal_service_controller.go | 6 +- ...teway_internal_service_enqueue_handlers.go | 17 +- .../gateway_pickup_controller.go | 6 +- .../gateway_pickup_enqueue_handlers.go | 16 +- .../gateway_public_service_controller.go | 4 +- ...gateway_public_service_enqueue_handlers.go | 16 +- .../endpoints/endpoints_controller.go | 13 +- .../endpoints/endpoints_enqueue_handlers.go | 10 +- .../endpointslice/endpointslice_controller.go | 32 +- .../endpointslice_enqueue_handlers.go | 10 +- .../controller/testutil/test_utils.go | 40 +- .../controller/util/node/controller_utils.go | 2 +- pkg/yurtmanager/controller/util/tools.go | 10 - .../nodepool_enqueue_handlers.go | 8 +- .../nodepool_enqueue_handlers_test.go | 9 +- .../yurtappdaemon/yurtappdaemon_controller.go | 4 +- .../yurtappoverrider_controller.go | 2 +- .../yurtappset/workloadmanager/util.go | 6 +- .../yurtappset/workloadmanager/util_test.go | 4 +- .../yurtappset/yurtappset_controller.go | 28 +- .../yurtappset/yurtappset_controller_test.go | 2 +- .../yurtcoordinator/cert/certificate.go | 4 +- .../controller/yurtcoordinator/cert/util.go | 4 +- .../cert/yurtcoordinatorcert_controller.go | 24 +- .../delegatelease/delegatelease_controller.go | 30 +- .../podbinding/podbinding_controller.go | 11 +- .../podbinding/podbinding_controller_test.go | 44 +- .../upgradeinfo/upgrade_info_test.go | 6 +- .../controller/yurtstaticset/util/util.go | 2 +- .../yurtstaticset/yurtstaticset_controller.go | 13 +- .../yurtstaticset_controller_test.go | 4 +- .../webhook/builder/defaulter_custom.go | 98 - .../webhook/builder/validator_custom.go | 112 - pkg/yurtmanager/webhook/builder/webhook.go | 199 -- .../gateway/v1alpha1/gateway_validation.go | 25 +- .../gateway/v1beta1/gateway_validation.go | 29 +- .../webhook/node/v1/node_default.go | 3 +- .../webhook/node/v1/node_default_test.go | 3 +- .../webhook/node/v1/node_handler.go | 8 +- .../webhook/node/v1/node_validation.go | 22 +- .../webhook/node/v1/node_validation_test.go | 3 +- .../nodepool/v1beta1/nodepool_validation.go | 27 +- .../v1beta1/nodepool_validation_test.go | 6 +- .../v1alpha1/platformadmin_validation.go | 23 +- .../v1alpha2/platformadmin_validation.go | 25 +- .../webhook/pod/v1alpha1/pod_default.go | 3 +- .../webhook/pod/v1alpha1/pod_default_test.go | 3 +- .../webhook/pod/v1alpha1/pod_handler.go | 6 +- .../util/writer/atomic/atomic_writer.go | 10 +- .../v1alpha1/yurtappdaemon_validation.go | 25 +- .../v1alpha1/yurtappoverrider_validation.go | 33 +- .../yurtappset/v1beta1/yurtappset_default.go | 2 +- .../v1beta1/yurtappset_validation.go | 41 +- .../v1beta1/yurtappset_webhook_test.go | 10 +- .../v1alpha1/yurtstaticset_validation.go | 23 +- pkg/yurttunnel/server/interceptor.go | 2 +- .../trafficforward/iptables/iptables.go | 4 +- .../trafficforward/iptables/iptables_test.go | 2 +- test/e2e/cmd/init/constants/constants.go | 1 - test/e2e/cmd/init/converter.go | 60 +- test/e2e/cmd/init/init.go | 18 +- .../cmd/init/util/kubernetes/apply_addons.go | 2 +- test/e2e/common/pod/pod.go | 8 +- test/e2e/util/nodepool.go | 4 +- test/e2e/util/util.go | 4 +- test/e2e/yurt/daemonpodupdater.go | 625 ++-- test/e2e/yurt/iot.go | 3 +- test/e2e/yurt/yurtappdaemon.go | 190 -- test/e2e/yurt/yurtappoverrider.go | 293 -- test/e2e/yurt/yurtstaticset.go | 143 +- 191 files changed, 6569 insertions(+), 5108 deletions(-) delete mode 100644 pkg/util/iptables/iptables_unsupported_test.go delete mode 100644 pkg/util/iptables/monitor_test.go create mode 100644 pkg/util/iptables/testing/parse.go create mode 100644 pkg/util/iptables/testing/parse_test.go delete mode 100644 pkg/yurtmanager/webhook/builder/defaulter_custom.go delete mode 100644 pkg/yurtmanager/webhook/builder/validator_custom.go delete mode 100644 pkg/yurtmanager/webhook/builder/webhook.go delete mode 100644 test/e2e/yurt/yurtappdaemon.go delete mode 100644 test/e2e/yurt/yurtappoverrider.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9665edce611..8ae3843f59b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,7 +9,8 @@ on: workflow_dispatch: {} env: - GO_VERSION: 1.18 + GO_VERSION: '1.20' + GOLANGCI_VERSION: 'v1.55.2' # Common users. We can't run a step 'if secrets.AWS_USR != ""' but we can run # a step 'if env.AWS_USR' != ""', so we copy these to succinctly test whether @@ -18,7 +19,6 @@ env: AWS_USR: ${{ secrets.AWS_USR }} jobs: - verify: runs-on: ubuntu-22.04 steps: @@ -42,11 +42,12 @@ jobs: with: go-version: ${{ env.GO_VERSION }} - name: Lint golang code - uses: golangci/golangci-lint-action@v3 + uses: golangci/golangci-lint-action@v6.0.1 with: - version: v1.54 - args: -v + version: ${{ env.GOLANGCI_VERSION }} + args: --verbose skip-cache: true + mode: readonly markdownlint-misspell-shellcheck: runs-on: ubuntu-22.04 @@ -120,8 +121,8 @@ jobs: # restore-keys: ${{ runner.os }}-go- - name: Install Required Commands run: | - go get sigs.k8s.io/kind@v0.12.0 - curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.22.3/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl + go install sigs.k8s.io/kind@v0.22.0 + curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.28.7/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl - name: Build Images run: make docker-build - name: Local Up Openyurt Cluster With Kind diff --git a/.gitignore b/.gitignore index 6f5593fc9f5..b6af30602ec 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ bin .DS_Store gopath dockerbuild +hack/cni vendor .vscode diff --git a/Makefile b/Makefile index 8626a704fe9..0b67a87f038 100644 --- a/Makefile +++ b/Makefile @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -KUBERNETESVERSION ?=v1.22 -GOLANGCILINT_VERSION ?= v1.54 +KUBERNETESVERSION ?=v1.28 +GOLANGCILINT_VERSION ?= v1.55.2 GLOBAL_GOLANGCILINT := $(shell which golangci-lint) GOBIN := $(shell go env GOPATH)/bin GOBIN_GOLANGCILINT := $(shell which $(GOBIN)/golangci-lint) @@ -59,7 +59,7 @@ KUSTOMIZE_VERSION ?= v4.5.7 ## Tool Binaries KUSTOMIZE ?= $(LOCALBIN)/kustomize -KUBECTL_VERSION ?= v1.22.3 +KUBECTL_VERSION ?= v1.28.7 KUBECTL ?= $(LOCALBIN)/kubectl YQ_VERSION := 4.13.2 @@ -209,7 +209,7 @@ $(KUBECTL): $(LOCALBIN) echo "$(LOCALBIN)/kubectl version is not expected $(KUBECTL_VERSION). Removing it before installing."; \ rm -rf $(LOCALBIN)/kubectl; \ fi - test -s $(LOCALBIN)/kubectl || curl https://storage.googleapis.com/kubernetes-release/release/v1.22.3/bin/$(shell go env GOOS)/$(shell go env GOARCH)/kubectl -o $(KUBECTL) + test -s $(LOCALBIN)/kubectl || curl https://storage.googleapis.com/kubernetes-release/release/$(KUBECTL_VERSION)/bin/$(shell go env GOOS)/$(shell go env GOARCH)/kubectl -o $(KUBECTL) chmod +x $(KUBECTL) KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" diff --git a/charts/yurt-manager/crds/iot.openyurt.io_platformadmins.yaml b/charts/yurt-manager/crds/iot.openyurt.io_platformadmins.yaml index 0b980281c1b..65af525ab03 100644 --- a/charts/yurt-manager/crds/iot.openyurt.io_platformadmins.yaml +++ b/charts/yurt-manager/crds/iot.openyurt.io_platformadmins.yaml @@ -210,6 +210,8 @@ spec: type: object template: description: Template describes the pods that will be created. + The only allowed template.spec.restartPolicy value is + "Always". properties: metadata: description: 'Standard object''s metadata. More info: @@ -604,10 +606,7 @@ spec: and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches - all namespaces. This field is - beta-level and is only honored - when PodAffinityNamespaceSelector - feature is enabled. + all namespaces. properties: matchExpressions: description: matchExpressions @@ -684,7 +683,7 @@ spec: by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this - pod's namespace" + pod's namespace". items: type: string type: array @@ -813,9 +812,7 @@ spec: and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all - namespaces. This field is beta-level - and is only honored when PodAffinityNamespaceSelector - feature is enabled. + namespaces. properties: matchExpressions: description: matchExpressions @@ -883,7 +880,7 @@ spec: selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's - namespace" + namespace". items: type: string type: array @@ -1018,10 +1015,7 @@ spec: and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches - all namespaces. This field is - beta-level and is only honored - when PodAffinityNamespaceSelector - feature is enabled. + all namespaces. properties: matchExpressions: description: matchExpressions @@ -1098,7 +1092,7 @@ spec: by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this - pod's namespace" + pod's namespace". items: type: string type: array @@ -1227,9 +1221,7 @@ spec: and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all - namespaces. This field is beta-level - and is only honored when PodAffinityNamespaceSelector - feature is enabled. + namespaces. properties: matchExpressions: description: matchExpressions @@ -1297,7 +1289,7 @@ spec: selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's - namespace" + namespace". items: type: string type: array @@ -1335,7 +1327,7 @@ spec: properties: args: description: 'Arguments to the entrypoint. - The docker image''s CMD is used if this + The container image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference @@ -1352,7 +1344,7 @@ spec: type: array command: description: 'Entrypoint array. Not executed - within a shell. The docker image''s ENTRYPOINT + within a shell. The container image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable @@ -1552,7 +1544,7 @@ spec: type: object type: array image: - description: 'Docker image name. More info: + description: 'Container image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override @@ -1581,9 +1573,8 @@ spec: info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the - following should be specified. Exec - specifies the action to take. + description: Exec specifies the action + to take. properties: command: description: Command is the command @@ -1624,7 +1615,11 @@ spec: properties: name: description: The header - field name + field name. This will + be canonicalized upon + output, so case-variant + names will be understood + as the same header. type: string value: description: The header @@ -1658,11 +1653,12 @@ spec: - port type: object tcpSocket: - description: 'TCPSocket specifies - an action involving a TCP port. - TCP hooks not yet supported TODO: - implement a realistic TCP lifecycle - hook' + description: Deprecated. TCPSocket + is NOT supported as a LifecycleHandler + and kept for the backward compatibility. + There are no validation of this + field and lifecycle hooks will fail + in runtime when tcp handler is specified. properties: host: description: 'Optional: Host name @@ -1690,22 +1686,20 @@ spec: such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container - crashes or exits. The reason for termination - is passed to the handler. The Pod''s - termination grace period countdown begins - before the PreStop hooked is executed. - Regardless of the outcome of the handler, - the container will eventually terminate - within the Pod''s termination grace - period. Other management of the container - blocks until the hook completes or until - the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + crashes or exits. The Pod''s termination + grace period countdown begins before + the PreStop hook is executed. Regardless + of the outcome of the handler, the container + will eventually terminate within the + Pod''s termination grace period (unless + delayed by finalizers). Other management + of the container blocks until the hook + completes or until the termination grace + period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the - following should be specified. Exec - specifies the action to take. + description: Exec specifies the action + to take. properties: command: description: Command is the command @@ -1746,7 +1740,11 @@ spec: properties: name: description: The header - field name + field name. This will + be canonicalized upon + output, so case-variant + names will be understood + as the same header. type: string value: description: The header @@ -1780,11 +1778,12 @@ spec: - port type: object tcpSocket: - description: 'TCPSocket specifies - an action involving a TCP port. - TCP hooks not yet supported TODO: - implement a realistic TCP lifecycle - hook' + description: Deprecated. TCPSocket + is NOT supported as a LifecycleHandler + and kept for the backward compatibility. + There are no validation of this + field and lifecycle hooks will fail + in runtime when tcp handler is specified. properties: host: description: 'Optional: Host name @@ -1813,9 +1812,8 @@ spec: info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following - should be specified. Exec specifies - the action to take. + description: Exec specifies the action + to take. properties: command: description: Command is the command @@ -1840,6 +1838,26 @@ spec: 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name + of the service to place in the gRPC + HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the + default behavior is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. @@ -1861,7 +1879,10 @@ spec: properties: name: description: The header field - name + name. This will be canonicalized + upon output, so case-variant + names will be understood as + the same header. type: string value: description: The header field @@ -1913,10 +1934,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not - yet supported TODO: implement a realistic - TCP lifecycle hook' + description: TCPSocket specifies an action + involving a TCP port. properties: host: description: 'Optional: Host name @@ -1974,15 +1993,15 @@ spec: type: string ports: description: List of ports to expose from - the container. Exposing a port here gives - the system additional information about - the network connections a container uses, - but is primarily informational. Not specifying - a port here DOES NOT prevent that port from - being exposed. Any port which is listening - on the default "0.0.0.0" address inside - a container will be accessible from the - network. Cannot be updated. + the container. Not specifying a port here + DOES NOT prevent that port from being exposed. + Any port which is listening on the default + "0.0.0.0" address inside a container will + be accessible from the network. Modifying + this array with strategic merge patch may + corrupt the data. For more information See + https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. items: description: ContainerPort represents a network port in a single container. @@ -2035,9 +2054,8 @@ spec: Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following - should be specified. Exec specifies - the action to take. + description: Exec specifies the action + to take. properties: command: description: Command is the command @@ -2062,6 +2080,26 @@ spec: 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name + of the service to place in the gRPC + HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the + default behavior is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. @@ -2083,7 +2121,10 @@ spec: properties: name: description: The header field - name + name. This will be canonicalized + upon output, so case-variant + names will be understood as + the same header. type: string value: description: The header field @@ -2135,10 +2176,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not - yet supported TODO: implement a realistic - TCP lifecycle hook' + description: TCPSocket specifies an action + involving a TCP port. properties: host: description: 'Optional: Host name @@ -2188,11 +2227,60 @@ spec: format: int32 type: integer type: object + resizePolicy: + description: Resources resize policy for the + container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: 'Name of the resource to + which this resource resize policy + applies. Supported values: cpu, memory.' + type: string + restartPolicy: + description: Restart policy to apply + when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic resources: description: 'Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' properties: + claims: + description: "Claims lists the names of + resources, defined in spec.resourceClaims, + that are used by this container. \n + This is an alpha field and requires + enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. + It can only be set for containers." + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the + name of one entry in pod.spec.resourceClaims + of the Pod where this field is + used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -2216,9 +2304,33 @@ spec: If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined - value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object + restartPolicy: + description: 'RestartPolicy defines the restart + behavior of individual containers in a pod. + This field may only be set for init containers, + and the only allowed value is "Always". + For non-init containers or when this field + is not specified, the restart behavior is + defined by the Pod''s restart policy and + the container type. Setting the RestartPolicy + as "Always" for the init container will + have the following effect: this init container + will be continually restarted on exit until + all regular containers have terminated. + Once all regular containers have completed, + all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs + from normal init containers and is often + referred to as a "sidecar" container. Although + this init container still starts in the + init container sequence, it does not wait + for the container to complete before proceeding + to the next init container.' + type: string securityContext: description: 'SecurityContext defines the security options the container should be @@ -2234,13 +2346,17 @@ spec: flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged - 2) has CAP_SYS_ADMIN' + 2) has CAP_SYS_ADMIN Note that this + field cannot be set when spec.os.name + is windows.' type: boolean capabilities: description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted - by the container runtime. + by the container runtime. Note that + this field cannot be set when spec.os.name + is windows. properties: add: description: Added capabilities @@ -2261,7 +2377,9 @@ spec: description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on - the host. Defaults to false. + the host. Defaults to false. Note that + this field cannot be set when spec.os.name + is windows. type: boolean procMount: description: procMount denotes the type @@ -2270,12 +2388,14 @@ spec: uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature - flag to be enabled. + flag to be enabled. Note that this field + cannot be set when spec.os.name is windows. type: string readOnlyRootFilesystem: description: Whether this container has a read-only root filesystem. Default - is false. + is false. Note that this field cannot + be set when spec.os.name is windows. type: boolean runAsGroup: description: The GID to run the entrypoint @@ -2284,7 +2404,8 @@ spec: PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field + cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: @@ -2307,7 +2428,8 @@ spec: unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field + cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: @@ -2318,7 +2440,8 @@ spec: also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field + cannot be set when spec.os.name is windows. properties: level: description: Level is SELinux level @@ -2342,7 +2465,8 @@ spec: by this container. If seccomp options are provided at both the pod & container level, the container options override - the pod options. + the pod options. Note that this field + cannot be set when spec.os.name is windows. properties: localhostProfile: description: localhostProfile indicates @@ -2351,8 +2475,9 @@ spec: must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured - seccomp profile location. Must only - be set if type is "Localhost". + seccomp profile location. Must be + set if type is "Localhost". Must + NOT be set for any other type. type: string type: description: "type indicates which @@ -2374,6 +2499,8 @@ spec: will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when + spec.os.name is linux. properties: gmsaCredentialSpec: description: GMSACredentialSpec is @@ -2391,18 +2518,12 @@ spec: hostProcess: description: HostProcess determines if a container should be run as - a 'Host Process' container. This - field is alpha-level and will only - be honored by components that enable - the WindowsHostProcessContainers - feature flag. Setting this field - without the feature flag will result - in errors when validating the Pod. - All of a Pod's containers must have + a 'Host Process' container. All + of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess - containers). In addition, if HostProcess + containers). In addition, if HostProcess is true then HostNetwork must also be set to true. type: boolean @@ -2433,9 +2554,8 @@ spec: info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following - should be specified. Exec specifies - the action to take. + description: Exec specifies the action + to take. properties: command: description: Command is the command @@ -2460,6 +2580,26 @@ spec: 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name + of the service to place in the gRPC + HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the + default behavior is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. @@ -2481,7 +2621,10 @@ spec: properties: name: description: The header field - name + name. This will be canonicalized + upon output, so case-variant + names will be understood as + the same header. type: string value: description: The header field @@ -2533,10 +2676,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not - yet supported TODO: implement a realistic - TCP lifecycle hook' + description: TCPSocket specifies an action + involving a TCP port. properties: host: description: 'Optional: Host name @@ -2784,29 +2925,25 @@ spec: when creating a pod, and it cannot be modified by updating the pod spec. In order to add an ephemeral container to an existing pod, use the pod's ephemeralcontainers - subresource. This field is alpha-level and is - only honored by servers that enable the EphemeralContainers - feature. + subresource. items: - description: An EphemeralContainer is a container - that may be added temporarily to an existing - pod for user-initiated activities such as debugging. + description: "An EphemeralContainer is a temporary + container that you may add to an existing Pod + for user-initiated activities such as debugging. Ephemeral containers have no resource or scheduling guarantees, and they will not be restarted when - they exit or when a pod is removed or restarted. - If an ephemeral container causes a pod to exceed - its resource allocation, the pod may be evicted. - Ephemeral containers may not be added by directly - updating the pod spec. They must be added via - the pod's ephemeralcontainers subresource, and - they will appear in the pod spec once added. - This is an alpha feature enabled by the EphemeralContainers - feature flag. + they exit or when a Pod is removed or restarted. + The kubelet may evict a Pod if an ephemeral + container causes the Pod to exceed its resource + allocation. \n To add an ephemeral container, + use the ephemeralcontainers subresource of an + existing Pod. Ephemeral containers may not be + removed or restarted." properties: args: description: 'Arguments to the entrypoint. - The docker image''s CMD is used if this - is not provided. Variable references $(VAR_NAME) + The image''s CMD is used if this is not + provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double @@ -2822,7 +2959,7 @@ spec: type: array command: description: 'Entrypoint array. Not executed - within a shell. The docker image''s ENTRYPOINT + within a shell. The image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable @@ -3022,7 +3159,7 @@ spec: type: object type: array image: - description: 'Docker image name. More info: + description: 'Container image name. More info: https://kubernetes.io/docs/concepts/containers/images' type: string imagePullPolicy: @@ -3046,9 +3183,8 @@ spec: info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the - following should be specified. Exec - specifies the action to take. + description: Exec specifies the action + to take. properties: command: description: Command is the command @@ -3089,7 +3225,11 @@ spec: properties: name: description: The header - field name + field name. This will + be canonicalized upon + output, so case-variant + names will be understood + as the same header. type: string value: description: The header @@ -3123,11 +3263,12 @@ spec: - port type: object tcpSocket: - description: 'TCPSocket specifies - an action involving a TCP port. - TCP hooks not yet supported TODO: - implement a realistic TCP lifecycle - hook' + description: Deprecated. TCPSocket + is NOT supported as a LifecycleHandler + and kept for the backward compatibility. + There are no validation of this + field and lifecycle hooks will fail + in runtime when tcp handler is specified. properties: host: description: 'Optional: Host name @@ -3155,22 +3296,20 @@ spec: such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container - crashes or exits. The reason for termination - is passed to the handler. The Pod''s - termination grace period countdown begins - before the PreStop hooked is executed. - Regardless of the outcome of the handler, - the container will eventually terminate - within the Pod''s termination grace - period. Other management of the container - blocks until the hook completes or until - the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + crashes or exits. The Pod''s termination + grace period countdown begins before + the PreStop hook is executed. Regardless + of the outcome of the handler, the container + will eventually terminate within the + Pod''s termination grace period (unless + delayed by finalizers). Other management + of the container blocks until the hook + completes or until the termination grace + period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the - following should be specified. Exec - specifies the action to take. + description: Exec specifies the action + to take. properties: command: description: Command is the command @@ -3211,7 +3350,11 @@ spec: properties: name: description: The header - field name + field name. This will + be canonicalized upon + output, so case-variant + names will be understood + as the same header. type: string value: description: The header @@ -3245,11 +3388,12 @@ spec: - port type: object tcpSocket: - description: 'TCPSocket specifies - an action involving a TCP port. - TCP hooks not yet supported TODO: - implement a realistic TCP lifecycle - hook' + description: Deprecated. TCPSocket + is NOT supported as a LifecycleHandler + and kept for the backward compatibility. + There are no validation of this + field and lifecycle hooks will fail + in runtime when tcp handler is specified. properties: host: description: 'Optional: Host name @@ -3276,9 +3420,8 @@ spec: containers. properties: exec: - description: One and only one of the following - should be specified. Exec specifies - the action to take. + description: Exec specifies the action + to take. properties: command: description: Command is the command @@ -3303,6 +3446,26 @@ spec: 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name + of the service to place in the gRPC + HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the + default behavior is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. @@ -3324,7 +3487,10 @@ spec: properties: name: description: The header field - name + name. This will be canonicalized + upon output, so case-variant + names will be understood as + the same header. type: string value: description: The header field @@ -3376,10 +3542,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not - yet supported TODO: implement a realistic - TCP lifecycle hook' + description: TCPSocket specifies an action + involving a TCP port. properties: host: description: 'Optional: Host name @@ -3479,14 +3643,17 @@ spec: - containerPort type: object type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map readinessProbe: description: Probes are not allowed for ephemeral containers. properties: exec: - description: One and only one of the following - should be specified. Exec specifies - the action to take. + description: Exec specifies the action + to take. properties: command: description: Command is the command @@ -3511,6 +3678,26 @@ spec: 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name + of the service to place in the gRPC + HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the + default behavior is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. @@ -3532,7 +3719,10 @@ spec: properties: name: description: The header field - name + name. This will be canonicalized + upon output, so case-variant + names will be understood as + the same header. type: string value: description: The header field @@ -3584,10 +3774,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not - yet supported TODO: implement a realistic - TCP lifecycle hook' + description: TCPSocket specifies an action + involving a TCP port. properties: host: description: 'Optional: Host name @@ -3637,12 +3825,61 @@ spec: format: int32 type: integer type: object + resizePolicy: + description: Resources resize policy for the + container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: 'Name of the resource to + which this resource resize policy + applies. Supported values: cpu, memory.' + type: string + restartPolicy: + description: Restart policy to apply + when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic resources: description: Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources already allocated to the pod. properties: + claims: + description: "Claims lists the names of + resources, defined in spec.resourceClaims, + that are used by this container. \n + This is an alpha field and requires + enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. + It can only be set for containers." + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the + name of one entry in pod.spec.resourceClaims + of the Pod where this field is + used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -3666,9 +3903,17 @@ spec: If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined - value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object + restartPolicy: + description: Restart policy for the container + to manage the restart behavior of each container + within a pod. This may only be set for init + containers. You cannot set this field on + ephemeral containers. + type: string securityContext: description: 'Optional: SecurityContext defines the security options the ephemeral container @@ -3684,13 +3929,17 @@ spec: flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged - 2) has CAP_SYS_ADMIN' + 2) has CAP_SYS_ADMIN Note that this + field cannot be set when spec.os.name + is windows.' type: boolean capabilities: description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted - by the container runtime. + by the container runtime. Note that + this field cannot be set when spec.os.name + is windows. properties: add: description: Added capabilities @@ -3711,7 +3960,9 @@ spec: description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on - the host. Defaults to false. + the host. Defaults to false. Note that + this field cannot be set when spec.os.name + is windows. type: boolean procMount: description: procMount denotes the type @@ -3720,12 +3971,14 @@ spec: uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature - flag to be enabled. + flag to be enabled. Note that this field + cannot be set when spec.os.name is windows. type: string readOnlyRootFilesystem: description: Whether this container has a read-only root filesystem. Default - is false. + is false. Note that this field cannot + be set when spec.os.name is windows. type: boolean runAsGroup: description: The GID to run the entrypoint @@ -3734,7 +3987,8 @@ spec: PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field + cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: @@ -3757,7 +4011,8 @@ spec: unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field + cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: @@ -3768,7 +4023,8 @@ spec: also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field + cannot be set when spec.os.name is windows. properties: level: description: Level is SELinux level @@ -3792,7 +4048,8 @@ spec: by this container. If seccomp options are provided at both the pod & container level, the container options override - the pod options. + the pod options. Note that this field + cannot be set when spec.os.name is windows. properties: localhostProfile: description: localhostProfile indicates @@ -3801,8 +4058,9 @@ spec: must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured - seccomp profile location. Must only - be set if type is "Localhost". + seccomp profile location. Must be + set if type is "Localhost". Must + NOT be set for any other type. type: string type: description: "type indicates which @@ -3824,6 +4082,8 @@ spec: will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when + spec.os.name is linux. properties: gmsaCredentialSpec: description: GMSACredentialSpec is @@ -3841,18 +4101,12 @@ spec: hostProcess: description: HostProcess determines if a container should be run as - a 'Host Process' container. This - field is alpha-level and will only - be honored by components that enable - the WindowsHostProcessContainers - feature flag. Setting this field - without the feature flag will result - in errors when validating the Pod. - All of a Pod's containers must have + a 'Host Process' container. All + of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess - containers). In addition, if HostProcess + containers). In addition, if HostProcess is true then HostNetwork must also be set to true. type: boolean @@ -3873,9 +4127,8 @@ spec: containers. properties: exec: - description: One and only one of the following - should be specified. Exec specifies - the action to take. + description: Exec specifies the action + to take. properties: command: description: Command is the command @@ -3900,6 +4153,26 @@ spec: 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name + of the service to place in the gRPC + HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the + default behavior is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. @@ -3921,7 +4194,10 @@ spec: properties: name: description: The header field - name + name. This will be canonicalized + upon output, so case-variant + names will be understood as + the same header. type: string value: description: The header field @@ -3973,10 +4249,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not - yet supported TODO: implement a realistic - TCP lifecycle hook' + description: TCPSocket specifies an action + involving a TCP port. properties: host: description: 'Optional: Host name @@ -4050,14 +4324,17 @@ spec: an EOF. Default is false type: boolean targetContainerName: - description: If set, the name of the container + description: "If set, the name of the container from PodSpec that this ephemeral container targets. The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. If not set then the ephemeral - container is run in whatever namespaces - are shared for the pod. Note that the container - runtime must support this feature. + container uses the namespaces configured + in the Pod spec. \n The container runtime + must implement support for this feature. + If the runtime does not support namespace + targeting then the result of setting this + field is undefined." type: string terminationMessagePath: description: 'Optional: Path at which the @@ -4113,7 +4390,9 @@ spec: type: array volumeMounts: description: Pod volumes to mount into the - container's filesystem. Cannot be updated. + container's filesystem. Subpath mounts are + not allowed for ephemeral containers. Cannot + be updated. items: description: VolumeMount describes a mounting of a Volume within a container. @@ -4204,6 +4483,21 @@ spec: description: 'Use the host''s pid namespace. Optional: Default to false.' type: boolean + hostUsers: + description: 'Use the host''s user namespace. Optional: + Default to true. If set to true or not present, + the pod will be run in the host user namespace, + useful for when the pod needs a feature only available + to the host user namespace, such as loading a + kernel module with CAP_SYS_MODULE. When set to + false, a new userns is created for the pod. Setting + false is useful for mitigating container breakout + vulnerabilities even allowing users to run their + containers as root without actually having root + privileges on the host. This field is alpha-level + and is only honored by servers that enable the + UserNamespacesSupport feature.' + type: boolean hostname: description: Specifies the hostname of the Pod If not specified, the pod's hostname will be set @@ -4215,9 +4509,7 @@ spec: to use for pulling any of the images used by this PodSpec. If specified, these secrets will be passed to individual puller implementations for them - to use. For example, in the case of docker, only - DockerConfig type secrets are honored. More info: - https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod' + to use. More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod' items: description: LocalObjectReference contains enough information to let you locate the referenced @@ -4255,7 +4547,7 @@ spec: properties: args: description: 'Arguments to the entrypoint. - The docker image''s CMD is used if this + The container image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference @@ -4272,7 +4564,7 @@ spec: type: array command: description: 'Entrypoint array. Not executed - within a shell. The docker image''s ENTRYPOINT + within a shell. The container image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable @@ -4472,7 +4764,7 @@ spec: type: object type: array image: - description: 'Docker image name. More info: + description: 'Container image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override @@ -4501,9 +4793,8 @@ spec: info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the - following should be specified. Exec - specifies the action to take. + description: Exec specifies the action + to take. properties: command: description: Command is the command @@ -4544,7 +4835,11 @@ spec: properties: name: description: The header - field name + field name. This will + be canonicalized upon + output, so case-variant + names will be understood + as the same header. type: string value: description: The header @@ -4578,11 +4873,12 @@ spec: - port type: object tcpSocket: - description: 'TCPSocket specifies - an action involving a TCP port. - TCP hooks not yet supported TODO: - implement a realistic TCP lifecycle - hook' + description: Deprecated. TCPSocket + is NOT supported as a LifecycleHandler + and kept for the backward compatibility. + There are no validation of this + field and lifecycle hooks will fail + in runtime when tcp handler is specified. properties: host: description: 'Optional: Host name @@ -4610,22 +4906,20 @@ spec: such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container - crashes or exits. The reason for termination - is passed to the handler. The Pod''s - termination grace period countdown begins - before the PreStop hooked is executed. - Regardless of the outcome of the handler, - the container will eventually terminate - within the Pod''s termination grace - period. Other management of the container - blocks until the hook completes or until - the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + crashes or exits. The Pod''s termination + grace period countdown begins before + the PreStop hook is executed. Regardless + of the outcome of the handler, the container + will eventually terminate within the + Pod''s termination grace period (unless + delayed by finalizers). Other management + of the container blocks until the hook + completes or until the termination grace + period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the - following should be specified. Exec - specifies the action to take. + description: Exec specifies the action + to take. properties: command: description: Command is the command @@ -4666,7 +4960,11 @@ spec: properties: name: description: The header - field name + field name. This will + be canonicalized upon + output, so case-variant + names will be understood + as the same header. type: string value: description: The header @@ -4700,11 +4998,12 @@ spec: - port type: object tcpSocket: - description: 'TCPSocket specifies - an action involving a TCP port. - TCP hooks not yet supported TODO: - implement a realistic TCP lifecycle - hook' + description: Deprecated. TCPSocket + is NOT supported as a LifecycleHandler + and kept for the backward compatibility. + There are no validation of this + field and lifecycle hooks will fail + in runtime when tcp handler is specified. properties: host: description: 'Optional: Host name @@ -4733,9 +5032,8 @@ spec: info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following - should be specified. Exec specifies - the action to take. + description: Exec specifies the action + to take. properties: command: description: Command is the command @@ -4760,6 +5058,26 @@ spec: 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name + of the service to place in the gRPC + HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the + default behavior is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. @@ -4781,7 +5099,10 @@ spec: properties: name: description: The header field - name + name. This will be canonicalized + upon output, so case-variant + names will be understood as + the same header. type: string value: description: The header field @@ -4833,10 +5154,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not - yet supported TODO: implement a realistic - TCP lifecycle hook' + description: TCPSocket specifies an action + involving a TCP port. properties: host: description: 'Optional: Host name @@ -4894,15 +5213,15 @@ spec: type: string ports: description: List of ports to expose from - the container. Exposing a port here gives - the system additional information about - the network connections a container uses, - but is primarily informational. Not specifying - a port here DOES NOT prevent that port from - being exposed. Any port which is listening - on the default "0.0.0.0" address inside - a container will be accessible from the - network. Cannot be updated. + the container. Not specifying a port here + DOES NOT prevent that port from being exposed. + Any port which is listening on the default + "0.0.0.0" address inside a container will + be accessible from the network. Modifying + this array with strategic merge patch may + corrupt the data. For more information See + https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. items: description: ContainerPort represents a network port in a single container. @@ -4955,9 +5274,8 @@ spec: Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following - should be specified. Exec specifies - the action to take. + description: Exec specifies the action + to take. properties: command: description: Command is the command @@ -4982,6 +5300,26 @@ spec: 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name + of the service to place in the gRPC + HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the + default behavior is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. @@ -5003,7 +5341,10 @@ spec: properties: name: description: The header field - name + name. This will be canonicalized + upon output, so case-variant + names will be understood as + the same header. type: string value: description: The header field @@ -5055,10 +5396,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not - yet supported TODO: implement a realistic - TCP lifecycle hook' + description: TCPSocket specifies an action + involving a TCP port. properties: host: description: 'Optional: Host name @@ -5108,11 +5447,60 @@ spec: format: int32 type: integer type: object + resizePolicy: + description: Resources resize policy for the + container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: 'Name of the resource to + which this resource resize policy + applies. Supported values: cpu, memory.' + type: string + restartPolicy: + description: Restart policy to apply + when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic resources: description: 'Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' properties: + claims: + description: "Claims lists the names of + resources, defined in spec.resourceClaims, + that are used by this container. \n + This is an alpha field and requires + enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. + It can only be set for containers." + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the + name of one entry in pod.spec.resourceClaims + of the Pod where this field is + used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -5136,9 +5524,33 @@ spec: If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined - value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object + restartPolicy: + description: 'RestartPolicy defines the restart + behavior of individual containers in a pod. + This field may only be set for init containers, + and the only allowed value is "Always". + For non-init containers or when this field + is not specified, the restart behavior is + defined by the Pod''s restart policy and + the container type. Setting the RestartPolicy + as "Always" for the init container will + have the following effect: this init container + will be continually restarted on exit until + all regular containers have terminated. + Once all regular containers have completed, + all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs + from normal init containers and is often + referred to as a "sidecar" container. Although + this init container still starts in the + init container sequence, it does not wait + for the container to complete before proceeding + to the next init container.' + type: string securityContext: description: 'SecurityContext defines the security options the container should be @@ -5154,13 +5566,17 @@ spec: flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged - 2) has CAP_SYS_ADMIN' + 2) has CAP_SYS_ADMIN Note that this + field cannot be set when spec.os.name + is windows.' type: boolean capabilities: description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted - by the container runtime. + by the container runtime. Note that + this field cannot be set when spec.os.name + is windows. properties: add: description: Added capabilities @@ -5181,7 +5597,9 @@ spec: description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on - the host. Defaults to false. + the host. Defaults to false. Note that + this field cannot be set when spec.os.name + is windows. type: boolean procMount: description: procMount denotes the type @@ -5190,12 +5608,14 @@ spec: uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature - flag to be enabled. + flag to be enabled. Note that this field + cannot be set when spec.os.name is windows. type: string readOnlyRootFilesystem: description: Whether this container has a read-only root filesystem. Default - is false. + is false. Note that this field cannot + be set when spec.os.name is windows. type: boolean runAsGroup: description: The GID to run the entrypoint @@ -5204,7 +5624,8 @@ spec: PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field + cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: @@ -5227,7 +5648,8 @@ spec: unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field + cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: @@ -5238,7 +5660,8 @@ spec: also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field + cannot be set when spec.os.name is windows. properties: level: description: Level is SELinux level @@ -5262,7 +5685,8 @@ spec: by this container. If seccomp options are provided at both the pod & container level, the container options override - the pod options. + the pod options. Note that this field + cannot be set when spec.os.name is windows. properties: localhostProfile: description: localhostProfile indicates @@ -5271,8 +5695,9 @@ spec: must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured - seccomp profile location. Must only - be set if type is "Localhost". + seccomp profile location. Must be + set if type is "Localhost". Must + NOT be set for any other type. type: string type: description: "type indicates which @@ -5294,6 +5719,8 @@ spec: will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when + spec.os.name is linux. properties: gmsaCredentialSpec: description: GMSACredentialSpec is @@ -5311,18 +5738,12 @@ spec: hostProcess: description: HostProcess determines if a container should be run as - a 'Host Process' container. This - field is alpha-level and will only - be honored by components that enable - the WindowsHostProcessContainers - feature flag. Setting this field - without the feature flag will result - in errors when validating the Pod. - All of a Pod's containers must have + a 'Host Process' container. All + of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess - containers). In addition, if HostProcess + containers). In addition, if HostProcess is true then HostNetwork must also be set to true. type: boolean @@ -5353,9 +5774,8 @@ spec: info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following - should be specified. Exec specifies - the action to take. + description: Exec specifies the action + to take. properties: command: description: Command is the command @@ -5380,6 +5800,26 @@ spec: 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name + of the service to place in the gRPC + HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the + default behavior is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. @@ -5401,7 +5841,10 @@ spec: properties: name: description: The header field - name + name. This will be canonicalized + upon output, so case-variant + names will be understood as + the same header. type: string value: description: The header field @@ -5453,10 +5896,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not - yet supported TODO: implement a realistic - TCP lifecycle hook' + description: TCPSocket specifies an action + involving a TCP port. properties: host: description: 'Optional: Host name @@ -5655,6 +6096,38 @@ spec: to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' type: object x-kubernetes-map-type: atomic + os: + description: "Specifies the OS of the containers + in the pod. Some pod and container fields are + restricted if this is set. \n If the OS field + is set to linux, the following fields must be + unset: -securityContext.windowsOptions \n If the + OS field is set to windows, following fields must + be unset: - spec.hostPID - spec.hostIPC - spec.hostUsers + - spec.securityContext.seLinuxOptions - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls - spec.shareProcessNamespace + - spec.securityContext.runAsUser - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*]." + properties: + name: + description: 'Name is the name of the operating + system. The currently supported values are + linux and windows. Additional value may be + defined in future and can be one of: https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional + values and treat unrecognized values in this + field as os: null' + type: string + required: + - name + type: object overhead: additionalProperties: anyOf: @@ -5674,17 +6147,13 @@ spec: in the PodSpec, Overhead will be set to the value defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. More - info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md - This field is beta-level as of Kubernetes v1.18, - and is only honored by servers that enable the - PodOverhead feature.' + info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md' type: object preemptionPolicy: description: PreemptionPolicy is the Policy for preempting pods with lower priority. One of Never, PreemptLowerPriority. Defaults to PreemptLowerPriority - if unset. This field is beta-level, gated by the - NonPreemptingPriority feature-gate. + if unset. type: string priority: description: The priority value. Various system @@ -5725,10 +6194,68 @@ spec: - conditionType type: object type: array + resourceClaims: + description: "ResourceClaims defines which ResourceClaims + must be allocated and reserved before the Pod + is allowed to start. The resources will be made + available to those containers which consume them + by name. \n This is an alpha field and requires + enabling the DynamicResourceAllocation feature + gate. \n This field is immutable." + items: + description: PodResourceClaim references exactly + one ResourceClaim through a ClaimSource. It + adds a name to it that uniquely identifies the + ResourceClaim inside the Pod. Containers that + need access to the ResourceClaim reference it + with this name. + properties: + name: + description: Name uniquely identifies this + resource claim inside the pod. This must + be a DNS_LABEL. + type: string + source: + description: Source describes where to find + the ResourceClaim. + properties: + resourceClaimName: + description: ResourceClaimName is the + name of a ResourceClaim object in the + same namespace as this pod. + type: string + resourceClaimTemplateName: + description: "ResourceClaimTemplateName + is the name of a ResourceClaimTemplate + object in the same namespace as this + pod. \n The template will be used to + create a new ResourceClaim, which will + be bound to this pod. When this pod + is deleted, the ResourceClaim will also + be deleted. The pod name and resource + name, along with a generated component, + will be used to form a unique name for + the ResourceClaim, which will be recorded + in pod.status.resourceClaimStatuses. + \n This field is immutable and no changes + will be made to the corresponding ResourceClaim + by the control plane after creating + the ResourceClaim." + type: string + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map restartPolicy: description: 'Restart policy for all containers within the pod. One of Always, OnFailure, Never. - Default to Always. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy' + In some contexts, only a subset of those values + may be permitted. Default to Always. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy' type: string runtimeClassName: description: 'RuntimeClassName refers to a RuntimeClass @@ -5738,14 +6265,39 @@ spec: If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an empty definition that uses the default runtime handler. - More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class - This is a beta feature as of Kubernetes v1.14.' + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class' type: string schedulerName: description: If specified, the pod will be dispatched by specified scheduler. If not specified, the pod will be dispatched by default scheduler. type: string + schedulingGates: + description: "SchedulingGates is an opaque list + of values that if specified will block scheduling + the pod. If schedulingGates is not empty, the + pod will stay in the SchedulingGated state and + the scheduler will not attempt to schedule the + pod. \n SchedulingGates can only be set at pod + creation time, and be removed only afterwards. + \n This is a beta feature enabled by the PodSchedulingReadiness + feature gate." + items: + description: PodSchedulingGate is associated to + a Pod to guard its scheduling. + properties: + name: + description: Name of the scheduling gate. + Each scheduling gate must have a unique + name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map securityContext: description: 'SecurityContext holds pod-level security attributes and common container settings. Optional: @@ -5762,7 +6314,8 @@ spec: volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- \n If unset, the Kubelet will not modify the ownership - and permissions of any volume." + and permissions of any volume. Note that this + field cannot be set when spec.os.name is windows." format: int64 type: integer fsGroupChangePolicy: @@ -5774,7 +6327,9 @@ spec: It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are "OnRootMismatch" and "Always". - If not specified, "Always" is used.' + If not specified, "Always" is used. Note that + this field cannot be set when spec.os.name + is windows.' type: string runAsGroup: description: The GID to run the entrypoint of @@ -5782,7 +6337,8 @@ spec: if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes - precedence for that container. + precedence for that container. Note that this + field cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: @@ -5804,7 +6360,8 @@ spec: be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. + for that container. Note that this field cannot + be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: @@ -5814,7 +6371,8 @@ spec: for each container. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes - precedence for that container. + precedence for that container. Note that this + field cannot be set when spec.os.name is windows. properties: level: description: Level is SELinux level label @@ -5835,7 +6393,8 @@ spec: type: object seccompProfile: description: The seccomp options to use by the - containers in this pod. + containers in this pod. Note that this field + cannot be set when spec.os.name is windows. properties: localhostProfile: description: localhostProfile indicates @@ -5843,8 +6402,9 @@ spec: should be used. The profile must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured - seccomp profile location. Must only be - set if type is "Localhost". + seccomp profile location. Must be set + if type is "Localhost". Must NOT be set + for any other type. type: string type: description: "type indicates which kind @@ -5861,8 +6421,16 @@ spec: supplementalGroups: description: A list of groups applied to the first process run in each container, in addition - to the container's primary GID. If unspecified, - no groups will be added to any container. + to the container's primary GID, the fsGroup + (if specified), and group memberships defined + in the container image for the uid of the + container process. If unspecified, no additional + groups are added to any container. Note that + group memberships defined in the container + image for the uid of the container process + are still effective, even if they are not + included in this list. Note that this field + cannot be set when spec.os.name is windows. items: format: int64 type: integer @@ -5871,7 +6439,8 @@ spec: description: Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail - to launch. + to launch. Note that this field cannot be + set when spec.os.name is windows. items: description: Sysctl defines a kernel parameter to be set @@ -5893,7 +6462,9 @@ spec: within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in - SecurityContext takes precedence. + SecurityContext takes precedence. Note that + this field cannot be set when spec.os.name + is linux. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where @@ -5909,18 +6480,12 @@ spec: hostProcess: description: HostProcess determines if a container should be run as a 'Host Process' - container. This field is alpha-level and - will only be honored by components that - enable the WindowsHostProcessContainers - feature flag. Setting this field without - the feature flag will result in errors - when validating the Pod. All of a Pod's - containers must have the same effective - HostProcess value (it is not allowed to - have a mix of HostProcess containers and - non-HostProcess containers). In addition, - if HostProcess is true then HostNetwork - must also be set to true. + container. All of a Pod's containers must + have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess + containers and non-HostProcess containers). + In addition, if HostProcess is true then + HostNetwork must also be set to true. type: boolean runAsUserName: description: The UserName in Windows to @@ -6101,19 +6666,45 @@ spec: only "value". The requirements are ANDed. type: object type: object + matchLabelKeys: + description: "MatchLabelKeys is a set of pod + label keys to select the pods over which + spreading will be calculated. The keys are + used to lookup values from the incoming + pod labels, those key-value labels are ANDed + with labelSelector to select the group of + existing pods over which spreading will + be calculated for the incoming pod. The + same key is forbidden to exist in both MatchLabelKeys + and LabelSelector. MatchLabelKeys cannot + be set when LabelSelector isn't set. Keys + that don't exist in the incoming pod labels + will be ignored. A null or empty list means + only match against labelSelector. \n This + is a beta field and requires the MatchLabelKeysInPodTopologySpread + feature gate to be enabled (enabled by default)." + items: + type: string + type: array + x-kubernetes-list-type: atomic maxSkew: description: 'MaxSkew describes the degree to which pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference between the number of matching pods in the target - topology and the global minimum. For example, - in a 3-zone cluster, MaxSkew is set to 1, - and pods with the same labelSelector spread - as 1/1/0: | zone1 | zone2 | zone3 | | P | P | | - - if MaxSkew is 1, incoming pod can only - be scheduled to zone3 to become 1/1/1; scheduling - it onto zone1(zone2) would make the ActualSkew(2-0) + topology and the global minimum. The global + minimum is the minimum number of matching + pods in an eligible domain or zero if the + number of eligible domains is less than + MinDomains. For example, in a 3-zone cluster, + MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: In this case, + the global minimum is 1. | zone1 | zone2 + | zone3 | | P P | P P | P | - if + MaxSkew is 1, incoming pod can only be scheduled + to zone3 to become 2/2/2; scheduling it + onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, @@ -6122,6 +6713,60 @@ spec: field. Default value is 1 and 0 is not allowed.' format: int32 type: integer + minDomains: + description: "MinDomains indicates a minimum + number of eligible domains. When the number + of eligible domains with matching topology + keys is less than minDomains, Pod Topology + Spread treats \"global minimum\" as 0, and + then the calculation of Skew is performed. + And when the number of eligible domains + with matching topology keys equals or greater + than minDomains, this value has no effect + on scheduling. As a result, when the number + of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew + Pods to those domains. If value is nil, + the constraint behaves as if MinDomains + is equal to 1. Valid values are integers + greater than 0. When value is not nil, WhenUnsatisfiable + must be DoNotSchedule. \n For example, in + a 3-zone cluster, MaxSkew is set to 2, MinDomains + is set to 5 and pods with the same labelSelector + spread as 2/2/2: | zone1 | zone2 | zone3 + | | P P | P P | P P | The number of + domains is less than 5(MinDomains), so \"global + minimum\" is treated as 0." + format: int32 + type: integer + nodeAffinityPolicy: + description: "NodeAffinityPolicy indicates + how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. + Options are: - Honor: only nodes matching + nodeAffinity/nodeSelector are included in + the calculations. - Ignore: nodeAffinity/nodeSelector + are ignored. All nodes are included in the + calculations. \n If this value is nil, the + behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled + by the NodeInclusionPolicyInPodTopologySpread + feature flag." + type: string + nodeTaintsPolicy: + description: "NodeTaintsPolicy indicates how + we will treat node taints when calculating + pod topology spread skew. Options are: - + Honor: nodes without taints, along with + tainted nodes for which the incoming pod + has a toleration, are included. - Ignore: + node taints are ignored. All nodes are included. + \n If this value is nil, the behavior is + equivalent to the Ignore policy. This is + a beta-level feature default enabled by + the NodeInclusionPolicyInPodTopologySpread + feature flag." + type: string topologyKey: description: TopologyKey is the key of node labels. Nodes that have a label with this @@ -6129,7 +6774,16 @@ spec: to be in the same topology. We consider each as a "bucket", and try to put balanced number of pods into each - bucket. It's a required field. + bucket. We define a domain as a particular + instance of a topology. Also, we define + an eligible domain as a domain whose nodes + meet the requirements of nodeAffinityPolicy + and nodeTaintsPolicy. e.g. If TopologyKey + is "kubernetes.io/hostname", each Node is + a domain of that topology. And, if TopologyKey + is "topology.kubernetes.io/zone", each zone + is a domain of that topology. It's a required + field. type: string whenUnsatisfiable: description: 'WhenUnsatisfiable indicates @@ -6141,7 +6795,7 @@ spec: giving higher precedence to topologies that would help reduce the skew. A constraint is considered "Unsatisfiable" for an incoming - pod if and only if every possible node assigment + pod if and only if every possible node assignment for that pod would violate "MaxSkew" on some topology. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with @@ -6175,139 +6829,144 @@ spec: in the pod. properties: awsElasticBlockStore: - description: 'AWSElasticBlockStore represents + description: 'awsElasticBlockStore represents an AWS Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' properties: fsType: - description: 'Filesystem type of the volume - that you want to mount. Tip: Ensure - that the filesystem type is supported - by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. More info: - https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + description: 'fsType is the filesystem + type of the volume that you want to + mount. Tip: Ensure that the filesystem + type is supported by the host operating + system. Examples: "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if + unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore TODO: how do we prevent errors in the filesystem from compromising the machine' type: string partition: - description: 'The partition in the volume - that you want to mount. If omitted, - the default is to mount by volume name. - Examples: For volume /dev/sda1, you - specify the partition as "1". Similarly, - the volume partition for /dev/sda is - "0" (or you can leave the property empty).' + description: 'partition is the partition + in the volume that you want to mount. + If omitted, the default is to mount + by volume name. Examples: For volume + /dev/sda1, you specify the partition + as "1". Similarly, the volume partition + for /dev/sda is "0" (or you can leave + the property empty).' format: int32 type: integer readOnly: - description: 'Specify "true" to force - and set the ReadOnly property in VolumeMounts - to "true". If omitted, the default is - "false". More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + description: 'readOnly value true will + force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' type: boolean volumeID: - description: 'Unique ID of the persistent - disk resource in AWS (Amazon EBS volume). - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + description: 'volumeID is unique ID of + the persistent disk resource in AWS + (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' type: string required: - volumeID type: object azureDisk: - description: AzureDisk represents an Azure + description: azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. properties: cachingMode: - description: 'Host Caching mode: None, - Read Only, Read Write.' + description: 'cachingMode is the Host + Caching mode: None, Read Only, Read + Write.' type: string diskName: - description: The Name of the data disk - in the blob storage + description: diskName is the Name of the + data disk in the blob storage type: string diskURI: - description: The URI the data disk in - the blob storage + description: diskURI is the URI of data + disk in the blob storage type: string fsType: - description: Filesystem type to mount. - Must be a filesystem type supported - by the host operating system. Ex. "ext4", - "xfs", "ntfs". Implicitly inferred to - be "ext4" if unspecified. + description: fsType is Filesystem type + to mount. Must be a filesystem type + supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. type: string kind: - description: 'Expected values Shared: - multiple blob disks per storage account Dedicated: - single blob disk per storage account Managed: - azure managed data disk (only in managed - availability set). defaults to shared' + description: 'kind expected values are + Shared: multiple blob disks per storage + account Dedicated: single blob disk + per storage account Managed: azure + managed data disk (only in managed availability + set). defaults to shared' type: string readOnly: - description: Defaults to false (read/write). - ReadOnly here will force the ReadOnly - setting in VolumeMounts. + description: readOnly Defaults to false + (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. type: boolean required: - diskName - diskURI type: object azureFile: - description: AzureFile represents an Azure + description: azureFile represents an Azure File Service mount on the host and bind mount to the pod. properties: readOnly: - description: Defaults to false (read/write). - ReadOnly here will force the ReadOnly - setting in VolumeMounts. + description: readOnly defaults to false + (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. type: boolean secretName: - description: the name of secret that contains - Azure Storage Account Name and Key + description: secretName is the name of + secret that contains Azure Storage Account + Name and Key type: string shareName: - description: Share Name + description: shareName is the azure share + Name type: string required: - secretName - shareName type: object cephfs: - description: CephFS represents a Ceph FS mount + description: cephFS represents a Ceph FS mount on the host that shares a pod's lifetime properties: monitors: - description: 'Required: Monitors is a - collection of Ceph monitors More info: - https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + description: 'monitors is Required: Monitors + is a collection of Ceph monitors More + info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' items: type: string type: array path: - description: 'Optional: Used as the mounted - root, rather than the full Ceph tree, - default is /' + description: 'path is Optional: Used as + the mounted root, rather than the full + Ceph tree, default is /' type: string readOnly: - description: 'Optional: Defaults to false - (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. + description: 'readOnly is Optional: Defaults + to false (read/write). ReadOnly here + will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' type: boolean secretFile: - description: 'Optional: SecretFile is - the path to key ring for User, default - is /etc/ceph/user.secret More info: - https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + description: 'secretFile is Optional: + SecretFile is the path to key ring for + User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' type: string secretRef: - description: 'Optional: SecretRef is reference - to the authentication secret for User, - default is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + description: 'secretRef is Optional: SecretRef + is reference to the authentication secret + for User, default is empty. More info: + https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' properties: name: description: 'Name of the referent. @@ -6317,36 +6976,36 @@ spec: type: string type: object user: - description: 'Optional: User is the rados - user name, default is admin More info: - https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + description: 'user is optional: User is + the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' type: string required: - monitors type: object cinder: - description: 'Cinder represents a cinder volume + description: 'cinder represents a cinder volume attached and mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' properties: fsType: - description: 'Filesystem type to mount. - Must be a filesystem type supported - by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. More info: - https://examples.k8s.io/mysql-cinder-pd/README.md' + description: 'fsType is the filesystem + type to mount. Must be a filesystem + type supported by the host operating + system. Examples: "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if + unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' type: string readOnly: - description: 'Optional: Defaults to false + description: 'readOnly defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' type: boolean secretRef: - description: 'Optional: points to a secret - object containing parameters used to - connect to OpenStack.' + description: 'secretRef is optional: points + to a secret object containing parameters + used to connect to OpenStack.' properties: name: description: 'Name of the referent. @@ -6356,76 +7015,76 @@ spec: type: string type: object volumeID: - description: 'volume id used to identify + description: 'volumeID used to identify the volume in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' type: string required: - volumeID type: object configMap: - description: ConfigMap represents a configMap + description: configMap represents a configMap that should populate this volume properties: defaultMode: - description: 'Optional: mode bits used - to set permissions on created files - by default. Must be an octal value between - 0000 and 0777 or a decimal value between - 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal - values for mode bits. Defaults to 0644. - Directories within the path are not - affected by this setting. This might - be in conflict with other options that - affect the file mode, like fsGroup, - and the result can be other mode bits - set.' + description: 'defaultMode is optional: + mode bits used to set permissions on + created files by default. Must be an + octal value between 0000 and 0777 or + a decimal value between 0 and 511. YAML + accepts both octal and decimal values, + JSON requires decimal values for mode + bits. Defaults to 0644. Directories + within the path are not affected by + this setting. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can + be other mode bits set.' format: int32 type: integer items: - description: If unspecified, each key-value - pair in the Data field of the referenced - ConfigMap will be projected into the - volume as a file whose name is the key - and content is the value. If specified, - the listed keys will be projected into - the specified paths, and unlisted keys - will not be present. If a key is specified - which is not present in the ConfigMap, - the volume setup will error unless it - is marked optional. Paths must be relative - and may not contain the '..' path or - start with '..'. + description: items if unspecified, each + key-value pair in the Data field of + the referenced ConfigMap will be projected + into the volume as a file whose name + is the key and content is the value. + If specified, the listed keys will be + projected into the specified paths, + and unlisted keys will not be present. + If a key is specified which is not present + in the ConfigMap, the volume setup will + error unless it is marked optional. + Paths must be relative and may not contain + the '..' path or start with '..'. items: description: Maps a string key to a path within a volume. properties: key: - description: The key to project. + description: key is the key to project. type: string mode: - description: 'Optional: mode bits - used to set permissions on this - file. Must be an octal value between - 0000 and 0777 or a decimal value - between 0 and 511. YAML accepts - both octal and decimal values, - JSON requires decimal values for - mode bits. If not specified, the - volume defaultMode will be used. - This might be in conflict with - other options that affect the - file mode, like fsGroup, and the - result can be other mode bits - set.' + description: 'mode is Optional: + mode bits used to set permissions + on this file. Must be an octal + value between 0000 and 0777 or + a decimal value between 0 and + 511. YAML accepts both octal and + decimal values, JSON requires + decimal values for mode bits. + If not specified, the volume defaultMode + will be used. This might be in + conflict with other options that + affect the file mode, like fsGroup, + and the result can be other mode + bits set.' format: int32 type: integer path: - description: The relative path of - the file to map the key to. May - not be an absolute path. May not - contain the path element '..'. - May not start with the string + description: path is the relative + path of the file to map the key + to. May not be an absolute path. + May not contain the path element + '..'. May not start with the string '..'. type: string required: @@ -6440,30 +7099,30 @@ spec: kind, uid?' type: string optional: - description: Specify whether the ConfigMap - or its keys must be defined + description: optional specify whether + the ConfigMap or its keys must be defined type: boolean type: object csi: - description: CSI (Container Storage Interface) + description: csi (Container Storage Interface) represents ephemeral storage that is handled by certain external CSI drivers (Beta feature). properties: driver: - description: Driver is the name of the + description: driver is the name of the CSI driver that handles this volume. Consult with your admin for the correct name as registered in the cluster. type: string fsType: - description: Filesystem type to mount. - Ex. "ext4", "xfs", "ntfs". If not provided, - the empty value is passed to the associated + description: fsType to mount. Ex. "ext4", + "xfs", "ntfs". If not provided, the + empty value is passed to the associated CSI driver which will determine the default filesystem to apply. type: string nodePublishSecretRef: - description: NodePublishSecretRef is a + description: nodePublishSecretRef is a reference to the secret object containing sensitive information to pass to the CSI driver to complete the CSI NodePublishVolume @@ -6481,13 +7140,14 @@ spec: type: string type: object readOnly: - description: Specifies a read-only configuration - for the volume. Defaults to false (read/write). + description: readOnly specifies a read-only + configuration for the volume. Defaults + to false (read/write). type: boolean volumeAttributes: additionalProperties: type: string - description: VolumeAttributes stores driver-specific + description: volumeAttributes stores driver-specific properties that are passed to the CSI driver. Consult your driver's documentation for supported values. @@ -6496,7 +7156,7 @@ spec: - driver type: object downwardAPI: - description: DownwardAPI represents downward + description: downwardAPI represents downward API about the pod that should populate this volume properties: @@ -6605,36 +7265,37 @@ spec: type: array type: object emptyDir: - description: 'EmptyDir represents a temporary + description: 'emptyDir represents a temporary directory that shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' properties: medium: - description: 'What type of storage medium - should back this directory. The default - is "" which means to use the node''s - default medium. Must be an empty string - (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + description: 'medium represents what type + of storage medium should back this directory. + The default is "" which means to use + the node''s default medium. Must be + an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' type: string sizeLimit: anyOf: - type: integer - type: string - description: 'Total amount of local storage - required for this EmptyDir volume. The - size limit is also applicable for memory - medium. The maximum usage on memory - medium EmptyDir would be the minimum - value between the SizeLimit specified - here and the sum of memory limits of - all containers in a pod. The default - is nil which means that the limit is - undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + description: 'sizeLimit is the total amount + of local storage required for this EmptyDir + volume. The size limit is also applicable + for memory medium. The maximum usage + on memory medium EmptyDir would be the + minimum value between the SizeLimit + specified here and the sum of memory + limits of all containers in a pod. The + default is nil which means that the + limit is undefined. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object ephemeral: - description: "Ephemeral represents a volume + description: "ephemeral represents a volume that is handled by a cluster storage driver. The volume's lifecycle is tied to the pod that defines it - it will be created before @@ -6701,7 +7362,7 @@ spec: valid here. properties: accessModes: - description: 'AccessModes contains + description: 'accessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' @@ -6709,20 +7370,25 @@ spec: type: string type: array dataSource: - description: 'This field can be - used to specify either: * An - existing VolumeSnapshot object - (snapshot.storage.k8s.io/VolumeSnapshot) + description: 'dataSource field + can be used to specify either: + * An existing VolumeSnapshot + object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. - If the AnyVolumeDataSource feature - gate is enabled, this field - will always have the same contents - as the DataSourceRef field.' + When the AnyVolumeDataSource + feature gate is enabled, dataSource + contents will be copied to dataSourceRef, + and dataSourceRef contents will + be copied to dataSource when + dataSourceRef.namespace is not + specified. If the namespace + is specified, then dataSourceRef + will not be copied to dataSource.' properties: apiGroup: description: APIGroup is the @@ -6747,11 +7413,11 @@ spec: - name type: object dataSourceRef: - description: 'Specifies the object - from which to populate the volume - with data, if a non-empty volume - is desired. This may be any - local object from a non-empty + description: dataSourceRef specifies + the object from which to populate + the volume with data, if a non-empty + volume is desired. This may + be any object from a non-empty API group (non core object) or a PersistentVolumeClaim object. When this field is specified, @@ -6761,22 +7427,20 @@ spec: volume populator or dynamic provisioner. This field will replace the functionality of - the DataSource field and as + the dataSource field and as such if both fields are non-empty, they must have the same value. For backwards compatibility, - both fields (DataSource and - DataSourceRef) will be set to - the same value automatically - if one of them is empty and - the other is non-empty. There - are two important differences - between DataSource and DataSourceRef: - * While DataSource only allows - two specific types of objects, - DataSourceRef allows any non-core - object, as well as PersistentVolumeClaim - objects.' + when namespace isn't specified + in dataSourceRef, both fields + (dataSource and dataSourceRef) + will be set to the same value + automatically if one of them + is empty and the other is non-empty. + When namespace is specified + in dataSourceRef, dataSource + isn't set to the same value + and must be empty. properties: apiGroup: description: APIGroup is the @@ -6796,15 +7460,70 @@ spec: description: Name is the name of resource being referenced type: string + namespace: + description: Namespace is + the namespace of resource + being referenced Note that + when a namespace is specified, + a gateway.networking.k8s.io/ReferenceGrant + object is required in the + referent namespace to allow + that namespace's owner to + accept the reference. See + the ReferenceGrant documentation + for details. (Alpha) This + field requires the CrossNamespaceVolumeDataSource + feature gate to be enabled. + type: string required: - kind - name type: object resources: - description: 'Resources represents + description: 'resources represents the minimum resources the volume - should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + should have. If RecoverVolumeExpansionFailure + feature is enabled users are + allowed to specify resource + requirements that are lower + than previous value but must + still be higher than capacity + recorded in the status field + of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' properties: + claims: + description: "Claims lists + the names of resources, + defined in spec.resourceClaims, + that are used by this container. + \n This is an alpha field + and requires enabling the + DynamicResourceAllocation + feature gate. \n This field + is immutable. It can only + be set for containers." + items: + description: ResourceClaim + references one entry in + PodSpec.ResourceClaims. + properties: + name: + description: Name must + match the name of + one entry in pod.spec.resourceClaims + of the Pod where this + field is used. It + makes that resource + available inside a + container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -6831,12 +7550,14 @@ spec: it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined - value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + value. Requests cannot exceed + Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object selector: - description: A label query over - volumes to consider for binding. + description: selector is a label + query over volumes to consider + for binding. properties: matchExpressions: description: matchExpressions @@ -6904,7 +7625,8 @@ spec: type: object type: object storageClassName: - description: 'Name of the StorageClass + description: 'storageClassName + is the name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' type: string @@ -6916,7 +7638,7 @@ spec: in claim spec. type: string volumeName: - description: VolumeName is the + description: volumeName is the binding reference to the PersistentVolume backing this claim. type: string @@ -6926,79 +7648,82 @@ spec: type: object type: object fc: - description: FC represents a Fibre Channel + description: fc represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. properties: fsType: - description: 'Filesystem type to mount. - Must be a filesystem type supported - by the host operating system. Ex. "ext4", - "xfs", "ntfs". Implicitly inferred to - be "ext4" if unspecified. TODO: how - do we prevent errors in the filesystem - from compromising the machine' + description: 'fsType is the filesystem + type to mount. Must be a filesystem + type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. + TODO: how do we prevent errors in the + filesystem from compromising the machine' type: string lun: - description: 'Optional: FC target lun - number' + description: 'lun is Optional: FC target + lun number' format: int32 type: integer readOnly: - description: 'Optional: Defaults to false - (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts.' + description: 'readOnly is Optional: Defaults + to false (read/write). ReadOnly here + will force the ReadOnly setting in VolumeMounts.' type: boolean targetWWNs: - description: 'Optional: FC target worldwide - names (WWNs)' + description: 'targetWWNs is Optional: + FC target worldwide names (WWNs)' items: type: string type: array wwids: - description: 'Optional: FC volume world - wide identifiers (wwids) Either wwids - or combination of targetWWNs and lun - must be set, but not both simultaneously.' + description: 'wwids Optional: FC volume + world wide identifiers (wwids) Either + wwids or combination of targetWWNs and + lun must be set, but not both simultaneously.' items: type: string type: array type: object flexVolume: - description: FlexVolume represents a generic + description: flexVolume represents a generic volume resource that is provisioned/attached using an exec based plugin. properties: driver: - description: Driver is the name of the + description: driver is the name of the driver to use for this volume. type: string fsType: - description: Filesystem type to mount. - Must be a filesystem type supported - by the host operating system. Ex. "ext4", - "xfs", "ntfs". The default filesystem - depends on FlexVolume script. + description: fsType is the filesystem + type to mount. Must be a filesystem + type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". The + default filesystem depends on FlexVolume + script. type: string options: additionalProperties: type: string - description: 'Optional: Extra command - options if any.' + description: 'options is Optional: this + field holds extra command options if + any.' type: object readOnly: - description: 'Optional: Defaults to false - (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts.' + description: 'readOnly is Optional: defaults + to false (read/write). ReadOnly here + will force the ReadOnly setting in VolumeMounts.' type: boolean secretRef: - description: 'Optional: SecretRef is reference - to the secret object containing sensitive - information to pass to the plugin scripts. - This may be empty if no secret object - is specified. If the secret object contains - more than one secret, all secrets are - passed to the plugin scripts.' + description: 'secretRef is Optional: secretRef + is reference to the secret object containing + sensitive information to pass to the + plugin scripts. This may be empty if + no secret object is specified. If the + secret object contains more than one + secret, all secrets are passed to the + plugin scripts.' properties: name: description: 'Name of the referent. @@ -7011,56 +7736,58 @@ spec: - driver type: object flocker: - description: Flocker represents a Flocker + description: flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running properties: datasetName: - description: Name of the dataset stored - as metadata -> name on the dataset for - Flocker should be considered as deprecated + description: datasetName is Name of the + dataset stored as metadata -> name on + the dataset for Flocker should be considered + as deprecated type: string datasetUUID: - description: UUID of the dataset. This - is unique identifier of a Flocker dataset + description: datasetUUID is the UUID of + the dataset. This is unique identifier + of a Flocker dataset type: string type: object gcePersistentDisk: - description: 'GCEPersistentDisk represents + description: 'gcePersistentDisk represents a GCE Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' properties: fsType: - description: 'Filesystem type of the volume - that you want to mount. Tip: Ensure - that the filesystem type is supported - by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. More info: - https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + description: 'fsType is filesystem type + of the volume that you want to mount. + Tip: Ensure that the filesystem type + is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk TODO: how do we prevent errors in the filesystem from compromising the machine' type: string partition: - description: 'The partition in the volume - that you want to mount. If omitted, - the default is to mount by volume name. - Examples: For volume /dev/sda1, you - specify the partition as "1". Similarly, - the volume partition for /dev/sda is - "0" (or you can leave the property empty). - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + description: 'partition is the partition + in the volume that you want to mount. + If omitted, the default is to mount + by volume name. Examples: For volume + /dev/sda1, you specify the partition + as "1". Similarly, the volume partition + for /dev/sda is "0" (or you can leave + the property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' format: int32 type: integer pdName: - description: 'Unique name of the PD resource - in GCE. Used to identify the disk in - GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + description: 'pdName is unique name of + the PD resource in GCE. Used to identify + the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' type: string readOnly: - description: 'ReadOnly here will force + description: 'readOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' type: boolean @@ -7068,7 +7795,7 @@ spec: - pdName type: object gitRepo: - description: 'GitRepo represents a git repository + description: 'gitRepo represents a git repository at a particular revision. DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an EmptyDir into @@ -7077,40 +7804,40 @@ spec: container.' properties: directory: - description: Target directory name. Must - not contain or start with '..'. If - '.' is supplied, the volume directory - will be the git repository. Otherwise, + description: directory is the target directory + name. Must not contain or start with + '..'. If '.' is supplied, the volume + directory will be the git repository. Otherwise, if specified, the volume will contain the git repository in the subdirectory with the given name. type: string repository: - description: Repository URL + description: repository is the URL type: string revision: - description: Commit hash for the specified - revision. + description: revision is the commit hash + for the specified revision. type: string required: - repository type: object glusterfs: - description: 'Glusterfs represents a Glusterfs + description: 'glusterfs represents a Glusterfs mount on the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' properties: endpoints: - description: 'EndpointsName is the endpoint + description: 'endpoints is the endpoint name that details Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' type: string path: - description: 'Path is the Glusterfs volume + description: 'path is the Glusterfs volume path. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' type: string readOnly: - description: 'ReadOnly here will force + description: 'readOnly here will force the Glusterfs volume to be mounted with read-only permissions. Defaults to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' @@ -7120,7 +7847,7 @@ spec: - path type: object hostPath: - description: 'HostPath represents a pre-existing + description: 'hostPath represents a pre-existing file or directory on the host machine that is directly exposed to the container. This is generally used for system agents or other @@ -7132,78 +7859,82 @@ spec: not mount host directories as read/write.' properties: path: - description: 'Path of the directory on + description: 'path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' type: string type: - description: 'Type for HostPath Volume + description: 'type for HostPath Volume Defaults to "" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' type: string required: - path type: object iscsi: - description: 'ISCSI represents an ISCSI Disk + description: 'iscsi represents an ISCSI Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' properties: chapAuthDiscovery: - description: whether support iSCSI Discovery - CHAP authentication + description: chapAuthDiscovery defines + whether support iSCSI Discovery CHAP + authentication type: boolean chapAuthSession: - description: whether support iSCSI Session - CHAP authentication + description: chapAuthSession defines whether + support iSCSI Session CHAP authentication type: boolean fsType: - description: 'Filesystem type of the volume - that you want to mount. Tip: Ensure - that the filesystem type is supported - by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. More info: - https://kubernetes.io/docs/concepts/storage/volumes#iscsi + description: 'fsType is the filesystem + type of the volume that you want to + mount. Tip: Ensure that the filesystem + type is supported by the host operating + system. Examples: "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if + unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi TODO: how do we prevent errors in the filesystem from compromising the machine' type: string initiatorName: - description: Custom iSCSI Initiator Name. - If initiatorName is specified with iscsiInterface - simultaneously, new iSCSI interface - : will be - created for the connection. + description: initiatorName is the custom + iSCSI Initiator Name. If initiatorName + is specified with iscsiInterface simultaneously, + new iSCSI interface : will be created for the connection. type: string iqn: - description: Target iSCSI Qualified Name. + description: iqn is the target iSCSI Qualified + Name. type: string iscsiInterface: - description: iSCSI Interface Name that - uses an iSCSI transport. Defaults to - 'default' (tcp). + description: iscsiInterface is the interface + Name that uses an iSCSI transport. Defaults + to 'default' (tcp). type: string lun: - description: iSCSI Target Lun number. + description: lun represents iSCSI Target + Lun number. format: int32 type: integer portals: - description: iSCSI Target Portal List. - The portal is either an IP or ip_addr:port - if the port is other than default (typically - TCP ports 860 and 3260). + description: portals is the iSCSI Target + Portal List. The portal is either an + IP or ip_addr:port if the port is other + than default (typically TCP ports 860 + and 3260). items: type: string type: array readOnly: - description: ReadOnly here will force + description: readOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. type: boolean secretRef: - description: CHAP Secret for iSCSI target - and initiator authentication + description: secretRef is the CHAP Secret + for iSCSI target and initiator authentication properties: name: description: 'Name of the referent. @@ -7213,10 +7944,11 @@ spec: type: string type: object targetPortal: - description: iSCSI Target Portal. The - Portal is either an IP or ip_addr:port - if the port is other than default (typically - TCP ports 860 and 3260). + description: targetPortal is iSCSI Target + Portal. The Portal is either an IP or + ip_addr:port if the port is other than + default (typically TCP ports 860 and + 3260). type: string required: - iqn @@ -7224,26 +7956,27 @@ spec: - targetPortal type: object name: - description: 'Volume''s name. Must be a DNS_LABEL - and unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: 'name of the volume. Must be + a DNS_LABEL and unique within the pod. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string nfs: - description: 'NFS represents an NFS mount + description: 'nfs represents an NFS mount on the host that shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' properties: path: - description: 'Path that is exported by + description: 'path that is exported by the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' type: string readOnly: - description: 'ReadOnly here will force + description: 'readOnly here will force the NFS export to be mounted with read-only permissions. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' type: boolean server: - description: 'Server is the hostname or + description: 'server is the hostname or IP address of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' type: string @@ -7252,98 +7985,100 @@ spec: - server type: object persistentVolumeClaim: - description: 'PersistentVolumeClaimVolumeSource + description: 'persistentVolumeClaimVolumeSource represents a reference to a PersistentVolumeClaim in the same namespace. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' properties: claimName: - description: 'ClaimName is the name of + description: 'claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' type: string readOnly: - description: Will force the ReadOnly setting - in VolumeMounts. Default false. + description: readOnly Will force the ReadOnly + setting in VolumeMounts. Default false. type: boolean required: - claimName type: object photonPersistentDisk: - description: PhotonPersistentDisk represents + description: photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine properties: fsType: - description: Filesystem type to mount. - Must be a filesystem type supported - by the host operating system. Ex. "ext4", - "xfs", "ntfs". Implicitly inferred to - be "ext4" if unspecified. + description: fsType is the filesystem + type to mount. Must be a filesystem + type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. type: string pdID: - description: ID that identifies Photon - Controller persistent disk + description: pdID is the ID that identifies + Photon Controller persistent disk type: string required: - pdID type: object portworxVolume: - description: PortworxVolume represents a portworx + description: portworxVolume represents a portworx volume attached and mounted on kubelets host machine properties: fsType: - description: FSType represents the filesystem + description: fSType represents the filesystem type to mount Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. type: string readOnly: - description: Defaults to false (read/write). - ReadOnly here will force the ReadOnly - setting in VolumeMounts. + description: readOnly defaults to false + (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. type: boolean volumeID: - description: VolumeID uniquely identifies + description: volumeID uniquely identifies a Portworx volume type: string required: - volumeID type: object projected: - description: Items for all in one resources - secrets, configmaps, and downward API + description: projected items for all in one + resources secrets, configmaps, and downward + API properties: defaultMode: - description: Mode bits used to set permissions - on created files by default. Must be - an octal value between 0000 and 0777 - or a decimal value between 0 and 511. - YAML accepts both octal and decimal - values, JSON requires decimal values - for mode bits. Directories within the - path are not affected by this setting. - This might be in conflict with other - options that affect the file mode, like - fsGroup, and the result can be other - mode bits set. + description: defaultMode are the mode + bits used to set permissions on created + files by default. Must be an octal value + between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both + octal and decimal values, JSON requires + decimal values for mode bits. Directories + within the path are not affected by + this setting. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can + be other mode bits set. format: int32 type: integer sources: - description: list of volume projections + description: sources is the list of volume + projections items: description: Projection that may be projected along with other supported volume types properties: configMap: - description: information about the - configMap data to project + description: configMap information + about the configMap data to project properties: items: - description: If unspecified, + description: items if unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected @@ -7365,26 +8100,26 @@ spec: key to a path within a volume. properties: key: - description: The key to - project. + description: key is the + key to project. type: string mode: - description: 'Optional: - mode bits used to set - permissions on this - file. Must be an octal - value between 0000 and - 0777 or a decimal value - between 0 and 511. YAML - accepts both octal and - decimal values, JSON - requires decimal values - for mode bits. If not - specified, the volume - defaultMode will be - used. This might be - in conflict with other - options that affect + description: 'mode is + Optional: mode bits + used to set permissions + on this file. Must be + an octal value between + 0000 and 0777 or a decimal + value between 0 and + 511. YAML accepts both + octal and decimal values, + JSON requires decimal + values for mode bits. + If not specified, the + volume defaultMode will + be used. This might + be in conflict with + other options that affect the file mode, like fsGroup, and the result can be other mode bits @@ -7392,14 +8127,14 @@ spec: format: int32 type: integer path: - description: The relative - path of the file to - map the key to. May - not be an absolute path. - May not contain the - path element '..'. May - not start with the string - '..'. + description: path is the + relative path of the + file to map the key + to. May not be an absolute + path. May not contain + the path element '..'. + May not start with the + string '..'. type: string required: - key @@ -7413,14 +8148,15 @@ spec: apiVersion, kind, uid?' type: string optional: - description: Specify whether - the ConfigMap or its keys - must be defined + description: optional specify + whether the ConfigMap or its + keys must be defined type: boolean type: object downwardAPI: - description: information about the - downwardAPI data to project + description: downwardAPI information + about the downwardAPI data to + project properties: items: description: Items is a list @@ -7527,11 +8263,11 @@ spec: type: array type: object secret: - description: information about the - secret data to project + description: secret information + about the secret data to project properties: items: - description: If unspecified, + description: items if unspecified, each key-value pair in the Data field of the referenced Secret will be projected into @@ -7553,26 +8289,26 @@ spec: key to a path within a volume. properties: key: - description: The key to - project. + description: key is the + key to project. type: string mode: - description: 'Optional: - mode bits used to set - permissions on this - file. Must be an octal - value between 0000 and - 0777 or a decimal value - between 0 and 511. YAML - accepts both octal and - decimal values, JSON - requires decimal values - for mode bits. If not - specified, the volume - defaultMode will be - used. This might be - in conflict with other - options that affect + description: 'mode is + Optional: mode bits + used to set permissions + on this file. Must be + an octal value between + 0000 and 0777 or a decimal + value between 0 and + 511. YAML accepts both + octal and decimal values, + JSON requires decimal + values for mode bits. + If not specified, the + volume defaultMode will + be used. This might + be in conflict with + other options that affect the file mode, like fsGroup, and the result can be other mode bits @@ -7580,14 +8316,14 @@ spec: format: int32 type: integer path: - description: The relative - path of the file to - map the key to. May - not be an absolute path. - May not contain the - path element '..'. May - not start with the string - '..'. + description: path is the + relative path of the + file to map the key + to. May not be an absolute + path. May not contain + the path element '..'. + May not start with the + string '..'. type: string required: - key @@ -7601,17 +8337,18 @@ spec: apiVersion, kind, uid?' type: string optional: - description: Specify whether - the Secret or its key must - be defined + description: optional field + specify whether the Secret + or its key must be defined type: boolean type: object serviceAccountToken: - description: information about the - serviceAccountToken data to project + description: serviceAccountToken + is information about the serviceAccountToken + data to project properties: audience: - description: Audience is the + description: audience is the intended audience of the token. A recipient of a token must identify itself with an identifier @@ -7622,7 +8359,7 @@ spec: of the apiserver. type: string expirationSeconds: - description: ExpirationSeconds + description: expirationSeconds is the requested duration of validity of the service account token. As the token @@ -7640,7 +8377,7 @@ spec: format: int64 type: integer path: - description: Path is the path + description: path is the path relative to the mount point of the file to project the token into. @@ -7652,20 +8389,20 @@ spec: type: array type: object quobyte: - description: Quobyte represents a Quobyte + description: quobyte represents a Quobyte mount on the host that shares a pod's lifetime properties: group: - description: Group to map volume access + description: group to map volume access to Default is no group type: string readOnly: - description: ReadOnly here will force + description: readOnly here will force the Quobyte volume to be mounted with read-only permissions. Defaults to false. type: boolean registry: - description: Registry represents a single + description: registry represents a single or multiple Quobyte Registry services specified as a string as host:port pair (multiple entries are separated with @@ -7673,17 +8410,17 @@ spec: for volumes type: string tenant: - description: Tenant owning the given Quobyte + description: tenant owning the given Quobyte volume in the Backend Used with dynamically provisioned Quobyte volumes, value is set by the plugin type: string user: - description: User to map volume access + description: user to map volume access to Defaults to serivceaccount user type: string volume: - description: Volume is a string that references + description: volume is a string that references an already created Quobyte volume by name. type: string @@ -7692,47 +8429,47 @@ spec: - volume type: object rbd: - description: 'RBD represents a Rados Block + description: 'rbd represents a Rados Block Device mount on the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' properties: fsType: - description: 'Filesystem type of the volume - that you want to mount. Tip: Ensure - that the filesystem type is supported - by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. More info: - https://kubernetes.io/docs/concepts/storage/volumes#rbd + description: 'fsType is the filesystem + type of the volume that you want to + mount. Tip: Ensure that the filesystem + type is supported by the host operating + system. Examples: "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if + unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd TODO: how do we prevent errors in the filesystem from compromising the machine' type: string image: - description: 'The rados image name. More - info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + description: 'image is the rados image + name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string keyring: - description: 'Keyring is the path to key + description: 'keyring is the path to key ring for RBDUser. Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string monitors: - description: 'A collection of Ceph monitors. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + description: 'monitors is a collection + of Ceph monitors. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' items: type: string type: array pool: - description: 'The rados pool name. Default - is rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + description: 'pool is the rados pool name. + Default is rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string readOnly: - description: 'ReadOnly here will force + description: 'readOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: boolean secretRef: - description: 'SecretRef is name of the + description: 'secretRef is name of the authentication secret for RBDUser. If provided overrides keyring. Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' @@ -7745,39 +8482,41 @@ spec: type: string type: object user: - description: 'The rados user name. Default - is admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + description: 'user is the rados user name. + Default is admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string required: - image - monitors type: object scaleIO: - description: ScaleIO represents a ScaleIO + description: scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. properties: fsType: - description: Filesystem type to mount. - Must be a filesystem type supported - by the host operating system. Ex. "ext4", - "xfs", "ntfs". Default is "xfs". + description: fsType is the filesystem + type to mount. Must be a filesystem + type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Default + is "xfs". type: string gateway: - description: The host address of the ScaleIO - API Gateway. + description: gateway is the host address + of the ScaleIO API Gateway. type: string protectionDomain: - description: The name of the ScaleIO Protection - Domain for the configured storage. + description: protectionDomain is the name + of the ScaleIO Protection Domain for + the configured storage. type: string readOnly: - description: Defaults to false (read/write). - ReadOnly here will force the ReadOnly - setting in VolumeMounts. + description: readOnly Defaults to false + (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. type: boolean secretRef: - description: SecretRef references to the + description: secretRef references to the secret for ScaleIO user and other sensitive information. If this is not provided, Login operation will fail. @@ -7790,27 +8529,29 @@ spec: type: string type: object sslEnabled: - description: Flag to enable/disable SSL - communication with Gateway, default + description: sslEnabled Flag enable/disable + SSL communication with Gateway, default false type: boolean storageMode: - description: Indicates whether the storage - for a volume should be ThickProvisioned + description: storageMode indicates whether + the storage for a volume should be ThickProvisioned or ThinProvisioned. Default is ThinProvisioned. type: string storagePool: - description: The ScaleIO Storage Pool - associated with the protection domain. + description: storagePool is the ScaleIO + Storage Pool associated with the protection + domain. type: string system: - description: The name of the storage system - as configured in ScaleIO. + description: system is the name of the + storage system as configured in ScaleIO. type: string volumeName: - description: The name of a volume already - created in the ScaleIO system that is - associated with this volume source. + description: volumeName is the name of + a volume already created in the ScaleIO + system that is associated with this + volume source. type: string required: - gateway @@ -7818,70 +8559,70 @@ spec: - system type: object secret: - description: 'Secret represents a secret that + description: 'secret represents a secret that should populate this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' properties: defaultMode: - description: 'Optional: mode bits used - to set permissions on created files - by default. Must be an octal value between - 0000 and 0777 or a decimal value between - 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal - values for mode bits. Defaults to 0644. - Directories within the path are not - affected by this setting. This might - be in conflict with other options that - affect the file mode, like fsGroup, - and the result can be other mode bits - set.' + description: 'defaultMode is Optional: + mode bits used to set permissions on + created files by default. Must be an + octal value between 0000 and 0777 or + a decimal value between 0 and 511. YAML + accepts both octal and decimal values, + JSON requires decimal values for mode + bits. Defaults to 0644. Directories + within the path are not affected by + this setting. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can + be other mode bits set.' format: int32 type: integer items: - description: If unspecified, each key-value - pair in the Data field of the referenced - Secret will be projected into the volume - as a file whose name is the key and - content is the value. If specified, - the listed keys will be projected into - the specified paths, and unlisted keys - will not be present. If a key is specified - which is not present in the Secret, - the volume setup will error unless it - is marked optional. Paths must be relative - and may not contain the '..' path or - start with '..'. + description: items If unspecified, each + key-value pair in the Data field of + the referenced Secret will be projected + into the volume as a file whose name + is the key and content is the value. + If specified, the listed keys will be + projected into the specified paths, + and unlisted keys will not be present. + If a key is specified which is not present + in the Secret, the volume setup will + error unless it is marked optional. + Paths must be relative and may not contain + the '..' path or start with '..'. items: description: Maps a string key to a path within a volume. properties: key: - description: The key to project. + description: key is the key to project. type: string mode: - description: 'Optional: mode bits - used to set permissions on this - file. Must be an octal value between - 0000 and 0777 or a decimal value - between 0 and 511. YAML accepts - both octal and decimal values, - JSON requires decimal values for - mode bits. If not specified, the - volume defaultMode will be used. - This might be in conflict with - other options that affect the - file mode, like fsGroup, and the - result can be other mode bits - set.' + description: 'mode is Optional: + mode bits used to set permissions + on this file. Must be an octal + value between 0000 and 0777 or + a decimal value between 0 and + 511. YAML accepts both octal and + decimal values, JSON requires + decimal values for mode bits. + If not specified, the volume defaultMode + will be used. This might be in + conflict with other options that + affect the file mode, like fsGroup, + and the result can be other mode + bits set.' format: int32 type: integer path: - description: The relative path of - the file to map the key to. May - not be an absolute path. May not - contain the path element '..'. - May not start with the string + description: path is the relative + path of the file to map the key + to. May not be an absolute path. + May not contain the path element + '..'. May not start with the string '..'. type: string required: @@ -7890,34 +8631,34 @@ spec: type: object type: array optional: - description: Specify whether the Secret - or its keys must be defined + description: optional field specify whether + the Secret or its keys must be defined type: boolean secretName: - description: 'Name of the secret in the - pod''s namespace to use. More info: - https://kubernetes.io/docs/concepts/storage/volumes#secret' + description: 'secretName is the name of + the secret in the pod''s namespace to + use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' type: string type: object storageos: - description: StorageOS represents a StorageOS + description: storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. properties: fsType: - description: Filesystem type to mount. - Must be a filesystem type supported - by the host operating system. Ex. "ext4", - "xfs", "ntfs". Implicitly inferred to - be "ext4" if unspecified. + description: fsType is the filesystem + type to mount. Must be a filesystem + type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. type: string readOnly: - description: Defaults to false (read/write). - ReadOnly here will force the ReadOnly - setting in VolumeMounts. + description: readOnly defaults to false + (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. type: boolean secretRef: - description: SecretRef specifies the secret + description: secretRef specifies the secret to use for obtaining the StorageOS API credentials. If not specified, default values will be attempted. @@ -7930,12 +8671,12 @@ spec: type: string type: object volumeName: - description: VolumeName is the human-readable + description: volumeName is the human-readable name of the StorageOS volume. Volume names are only unique within a namespace. type: string volumeNamespace: - description: VolumeNamespace specifies + description: volumeNamespace specifies the scope of the volume within StorageOS. If no namespace is specified then the Pod's namespace will be used. This allows @@ -7949,29 +8690,30 @@ spec: type: string type: object vsphereVolume: - description: VsphereVolume represents a vSphere + description: vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine properties: fsType: - description: Filesystem type to mount. - Must be a filesystem type supported - by the host operating system. Ex. "ext4", - "xfs", "ntfs". Implicitly inferred to - be "ext4" if unspecified. + description: fsType is filesystem type + to mount. Must be a filesystem type + supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. type: string storagePolicyID: - description: Storage Policy Based Management - (SPBM) profile ID associated with the - StoragePolicyName. + description: storagePolicyID is the storage + Policy Based Management (SPBM) profile + ID associated with the StoragePolicyName. type: string storagePolicyName: - description: Storage Policy Based Management - (SPBM) profile name. + description: storagePolicyName is the + storage Policy Based Management (SPBM) + profile name. type: string volumePath: - description: Path that identifies vSphere - volume vmdk + description: volumePath is the path that + identifies vSphere volume vmdk type: string required: - volumePath @@ -8012,9 +8754,7 @@ spec: a value), those requests will be respected, regardless of this field. This field may only be set for services with type LoadBalancer and will be cleared if the type - is changed to any other type. This field is beta-level - and is only honored by servers that enable the ServiceLBNodePortControl - feature. + is changed to any other type. type: boolean clusterIP: description: clusterIP is the IP address of the service @@ -8075,14 +8815,23 @@ spec: and requires `type` to be "ExternalName". type: string externalTrafficPolicy: - description: externalTrafficPolicy denotes if this Service - desires to route external traffic to node-local or cluster-wide - endpoints. "Local" preserves the client source IP and - avoids a second hop for LoadBalancer and Nodeport type - services, but risks potentially imbalanced traffic spreading. - "Cluster" obscures the client source IP and may cause - a second hop to another node, but should have good overall - load-spreading. + description: externalTrafficPolicy describes how nodes distribute + service traffic they receive on one of the Service's "externally-facing" + addresses (NodePorts, ExternalIPs, and LoadBalancer IPs). + If set to "Local", the proxy will configure the service + in a way that assumes that external load balancers will + take care of balancing the service traffic between nodes, + and so each node will deliver traffic only to the node-local + endpoints of the service, without masquerading the client + source IP. (Traffic mistakenly sent to a node with no + endpoints will be dropped.) The default value, "Cluster", + uses the standard behavior of routing to all endpoints + evenly (possibly modified by topology and other features). + Note that traffic sent to an External IP or LoadBalancer + IP from within the cluster will always get "Cluster" semantics, + but clients sending to a NodePort from within the cluster + may need to take traffic policy into account when picking + a node. type: string healthCheckNodePort: description: healthCheckNodePort specifies the healthcheck @@ -8096,32 +8845,33 @@ spec: field is specified when creating a Service which does not need it, creation will fail. This field will be wiped when updating a Service to no longer need it (e.g. changing - type). + type). This field cannot be updated once set. format: int32 type: integer internalTrafficPolicy: - description: InternalTrafficPolicy specifies if the cluster - internal traffic should be routed to all endpoints or - node-local endpoints only. "Cluster" routes internal traffic - to a Service to all endpoints. "Local" routes traffic - to node-local endpoints only, traffic is dropped if no - node-local endpoints are ready. The default value is "Cluster". + description: InternalTrafficPolicy describes how nodes distribute + service traffic they receive on the ClusterIP. If set + to "Local", the proxy will assume that pods only want + to talk to endpoints of the service on the same node as + the pod, dropping the traffic if there are no local endpoints. + The default value, "Cluster", uses the standard behavior + of routing to all endpoints evenly (possibly modified + by topology and other features). type: string ipFamilies: description: "IPFamilies is a list of IP families (e.g. - IPv4, IPv6) assigned to this service, and is gated by - the \"IPv6DualStack\" feature gate. This field is usually + IPv4, IPv6) assigned to this service. This field is usually assigned automatically based on cluster configuration and the ipFamilyPolicy field. If this field is specified manually, the requested family is available in the cluster, and ipFamilyPolicy allows it, it will be used; otherwise - creation of the service will fail. This field is conditionally + creation of the service will fail. This field is conditionally mutable: it allows for adding or removing a secondary IP family, but it does not allow changing the primary - IP family of the Service. Valid values are \"IPv4\" and + IP family of the Service. Valid values are \"IPv4\" and \"IPv6\". This field only applies to Services of types ClusterIP, NodePort, and LoadBalancer, and does apply - to \"headless\" services. This field will be wiped when + to \"headless\" services. This field will be wiped when updating a Service to type ExternalName. \n This field may hold a maximum of two entries (dual-stack families, in either order). These families must correspond to the @@ -8135,15 +8885,14 @@ spec: x-kubernetes-list-type: atomic ipFamilyPolicy: description: IPFamilyPolicy represents the dual-stack-ness - requested or required by this Service, and is gated by - the "IPv6DualStack" feature gate. If there is no value - provided, then this field will be set to SingleStack. + requested or required by this Service. If there is no + value provided, then this field will be set to SingleStack. Services can be "SingleStack" (a single IP family), "PreferDualStack" (two IP families on dual-stack configured clusters or a single IP family on single-stack clusters), or "RequireDualStack" (two IP families on dual-stack configured clusters, otherwise fail). The ipFamilies and clusterIPs fields depend on - the value of this field. This field will be wiped when + the value of this field. This field will be wiped when updating a service to type ExternalName. type: string loadBalancerClass: @@ -8165,12 +8914,15 @@ spec: when a service is updated to a non 'LoadBalancer' type. type: string loadBalancerIP: - description: 'Only applies to Service Type: LoadBalancer - LoadBalancer will get created with the IP specified in - this field. This feature depends on whether the underlying - cloud-provider supports specifying the loadBalancerIP - when a load balancer is created. This field will be ignored - if the cloud-provider does not support the feature.' + description: 'Only applies to Service Type: LoadBalancer. + This feature depends on whether the underlying cloud-provider + supports specifying the loadBalancerIP when a load balancer + is created. This field will be ignored if the cloud-provider + does not support the feature. Deprecated: This field was + under-specified and its meaning varies across implementations. + Using it is non-portable and it may not support dual-stack. + Users are encouraged to use implementation-specific annotations + when available.' type: string loadBalancerSourceRanges: description: 'If specified and supported by the platform, @@ -8189,12 +8941,21 @@ spec: port. properties: appProtocol: - description: The application protocol for this port. + description: "The application protocol for this port. + This is used as a hint for implementations to offer + richer behavior for protocols that they understand. This field follows standard Kubernetes label syntax. - Un-prefixed names are reserved for IANA standard - service names (as per RFC-6335 and http://www.iana.org/assignments/service-names). - Non-standard protocols should use prefixed names - such as mycompany.com/my-custom-protocol. + Valid values are either: \n * Un-prefixed protocol + names - reserved for IANA standard service names + (as per RFC-6335 and https://www.iana.org/assignments/service-names). + \n * Kubernetes-defined prefixed names: * 'kubernetes.io/h2c' + - HTTP/2 over cleartext as described in https://www.rfc-editor.org/rfc/rfc7540 + \ * 'kubernetes.io/ws' - WebSocket over cleartext + as described in https://www.rfc-editor.org/rfc/rfc6455 + \ * 'kubernetes.io/wss' - WebSocket over TLS as + described in https://www.rfc-editor.org/rfc/rfc6455 + \n * Other protocols should use implementation-defined + prefixed names such as mycompany.com/my-custom-protocol." type: string name: description: The name of this port within the service. diff --git a/charts/yurt-manager/crds/network.openyurt.io_poolservices.yaml b/charts/yurt-manager/crds/network.openyurt.io_poolservices.yaml index c74184a537d..b12d05702c6 100644 --- a/charts/yurt-manager/crds/network.openyurt.io_poolservices.yaml +++ b/charts/yurt-manager/crds/network.openyurt.io_poolservices.yaml @@ -71,13 +71,14 @@ spec: description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions. For example, - type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: - \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type - \ // +patchStrategy=merge // +listType=map // +listMapKey=type - \ Conditions []metav1.Condition `json:\"conditions,omitempty\" - patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` - \n // other fields }" + \n \ttype FooStatus struct{ \t // Represents the observations + of a foo's current state. \t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\" \t // + +patchMergeKey=type \t // +patchStrategy=merge \t // +listType=map + \t // +listMapKey=type \t Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n \t // other fields + \t}" properties: lastTransitionTime: description: lastTransitionTime is the last time the condition diff --git a/cmd/yurt-iot-dock/app/core.go b/cmd/yurt-iot-dock/app/core.go index d889fd071b4..47ffc6f5727 100644 --- a/cmd/yurt-iot-dock/app/core.go +++ b/cmd/yurt-iot-dock/app/core.go @@ -19,6 +19,7 @@ package app import ( "context" "fmt" + "net/http" "os" "os/signal" "syscall" @@ -34,6 +35,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "github.com/openyurtio/openyurt/cmd/yurt-iot-dock/app/options" "github.com/openyurtio/openyurt/pkg/apis" @@ -81,13 +83,17 @@ func Run(opts *options.YurtIoTDockOptions, stopCh <-chan struct{}) { ctrl.SetLogger(klogr.New()) cfg := ctrl.GetConfigOrDie() + metricsServerOpts := metricsserver.Options{ + BindAddress: opts.MetricsAddr, + ExtraHandlers: make(map[string]http.Handler, 0), + } + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme, - MetricsBindAddress: opts.MetricsAddr, + Metrics: metricsServerOpts, HealthProbeBindAddress: opts.ProbeAddr, LeaderElection: opts.EnableLeaderElection, LeaderElectionID: "yurt-iot-dock", - Namespace: opts.Namespace, }) if err != nil { setupLog.Error(err, "unable to start manager") diff --git a/cmd/yurt-iot-dock/yurt-iot-dock.go b/cmd/yurt-iot-dock/yurt-iot-dock.go index c647e5241c0..4a7bc250187 100644 --- a/cmd/yurt-iot-dock/yurt-iot-dock.go +++ b/cmd/yurt-iot-dock/yurt-iot-dock.go @@ -18,8 +18,6 @@ package main import ( "flag" - "math/rand" - "time" "k8s.io/apimachinery/pkg/util/wait" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) @@ -31,7 +29,6 @@ import ( ) func main() { - rand.Seed(time.Now().UnixNano()) klog.InitFlags(nil) defer klog.Flush() diff --git a/cmd/yurt-manager/app/manager.go b/cmd/yurt-manager/app/manager.go index 361cd83d043..1e4e629e9d3 100644 --- a/cmd/yurt-manager/app/manager.go +++ b/cmd/yurt-manager/app/manager.go @@ -18,6 +18,7 @@ package app import ( "fmt" + "net/http" "os" "github.com/spf13/cobra" @@ -34,6 +35,8 @@ import ( "k8s.io/klog/v2/klogr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + runtimewebhook "sigs.k8s.io/controller-runtime/pkg/webhook" "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" "github.com/openyurtio/openyurt/cmd/yurt-manager/app/options" @@ -160,19 +163,28 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error { } setRestConfig(cfg, c) + metricsServerOpts := metricsserver.Options{ + BindAddress: c.ComponentConfig.Generic.MetricsAddr, + ExtraHandlers: make(map[string]http.Handler, 0), + } + for path, handler := range profile.GetPprofHandlers() { + metricsServerOpts.ExtraHandlers[path] = handler + } + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme, - MetricsBindAddress: c.ComponentConfig.Generic.MetricsAddr, + Metrics: metricsServerOpts, HealthProbeBindAddress: c.ComponentConfig.Generic.HealthProbeAddr, LeaderElection: c.ComponentConfig.Generic.LeaderElection.LeaderElect, LeaderElectionID: c.ComponentConfig.Generic.LeaderElection.ResourceName, LeaderElectionNamespace: c.ComponentConfig.Generic.LeaderElection.ResourceNamespace, LeaderElectionResourceLock: c.ComponentConfig.Generic.LeaderElection.ResourceLock, - Port: util.GetWebHookPort(), - Namespace: "", - Logger: setupLog, - CertDir: util.GetCertDir(), - Host: "0.0.0.0", + WebhookServer: runtimewebhook.NewServer(runtimewebhook.Options{ + Host: "0.0.0.0", + Port: util.GetWebHookPort(), + CertDir: util.GetCertDir(), + }), + Logger: setupLog, }) if err != nil { setupLog.Error(err, "unable to start manager") @@ -216,13 +228,6 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error { os.Exit(1) } - for path, handler := range profile.GetPprofHandlers() { - if err := mgr.AddMetricsExtraHandler(path, handler); err != nil { - setupLog.Error(err, "unable to add pprof handler") - os.Exit(1) - } - } - setupLog.Info("starting manager") if err := mgr.Start(ctx); err != nil { setupLog.Error(err, "problem running manager") diff --git a/cmd/yurt-tunnel-agent/app/start.go b/cmd/yurt-tunnel-agent/app/start.go index b64095a0fe1..2f5733a8ee4 100644 --- a/cmd/yurt-tunnel-agent/app/start.go +++ b/cmd/yurt-tunnel-agent/app/start.go @@ -17,6 +17,7 @@ limitations under the License. package app import ( + "context" "fmt" "net" "os" @@ -111,14 +112,14 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error { agentCertMgr.Start() // 2.1. waiting for the certificate is generated - _ = wait.PollUntil(5*time.Second, func() (bool, error) { + _ = wait.PollUntilContextCancel(context.Background(), 5*time.Second, true, func(ctx context.Context) (bool, error) { if agentCertMgr.Current() != nil { return true, nil } klog.Infof("certificate %s not signed, waiting...", projectinfo.GetAgentName()) return false, nil - }, stopCh) + }) klog.Infof("certificate %s ok", projectinfo.GetAgentName()) // 3. generate a TLS configuration for securing the connection to server diff --git a/cmd/yurt-tunnel-server/app/config/config.go b/cmd/yurt-tunnel-server/app/config/config.go index 73d5b03c977..42148eb585a 100644 --- a/cmd/yurt-tunnel-server/app/config/config.go +++ b/cmd/yurt-tunnel-server/app/config/config.go @@ -76,5 +76,5 @@ func (c *Config) Complete() *CompletedConfig { } func (c *Config) IsIPv6() bool { - return c.IPFamily == iptables.ProtocolIpv6 + return c.IPFamily == iptables.ProtocolIPv6 } diff --git a/cmd/yurt-tunnel-server/app/options/options.go b/cmd/yurt-tunnel-server/app/options/options.go index 1804396d596..f2bb6830286 100644 --- a/cmd/yurt-tunnel-server/app/options/options.go +++ b/cmd/yurt-tunnel-server/app/options/options.go @@ -140,9 +140,9 @@ func (o *ServerOptions) Config() (*config.Config, error) { } if utilnet.IsIPv6String(o.BindAddr) { - cfg.IPFamily = iptables.ProtocolIpv6 + cfg.IPFamily = iptables.ProtocolIPv6 } else { - cfg.IPFamily = iptables.ProtocolIpv4 + cfg.IPFamily = iptables.ProtocolIPv4 } cfg.ListenAddrForAgent = net.JoinHostPort(o.BindAddr, o.TunnelAgentConnectPort) cfg.ListenAddrForMaster = net.JoinHostPort(o.BindAddr, o.SecurePort) diff --git a/cmd/yurt-tunnel-server/app/start.go b/cmd/yurt-tunnel-server/app/start.go index 218c2558b12..6c57e858849 100644 --- a/cmd/yurt-tunnel-server/app/start.go +++ b/cmd/yurt-tunnel-server/app/start.go @@ -167,14 +167,14 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error { cfg.SharedInformerFactory.Start(stopCh) // 5. waiting for the certificate is generated - _ = wait.PollUntil(5*time.Second, func() (bool, error) { + _ = wait.PollUntilContextCancel(context.Background(), 5*time.Second, true, func(ctx context.Context) (bool, error) { // keep polling until the certificate is signed if serverCertMgr.Current() != nil && tunnelProxyCertMgr.Current() != nil { return true, nil } klog.Infof("waiting for the master to sign the %s certificate", projectinfo.GetServerName()) return false, nil - }, stopCh) + }) // 6. generate the TLS configuration based on the latest certificate tlsCfg, err := certmanager.GenTLSConfigUseCurrentCertAndCertPool(serverCertMgr.Current, cfg.RootCert, "server") @@ -219,7 +219,7 @@ func getTunnelServerIPsAndDNSNamesBeforeInformerSynced(clientset kubernetes.Inte ) // the ips and dnsNames should be acquired through api-server at the first time, because the informer factory has not started yet. - werr := wait.PollUntil(5*time.Second, func() (bool, error) { + werr := wait.PollUntilContextCancel(context.Background(), 5*time.Second, true, func(ctx context.Context) (bool, error) { dnsNames, ips, err = serveraddr.GetYurttunelServerDNSandIP(clientset) if err != nil { klog.Errorf("failed to get yurt tunnel server dns and ip, %v", err) @@ -243,7 +243,7 @@ func getTunnelServerIPsAndDNSNamesBeforeInformerSynced(clientset kubernetes.Inte } return true, nil - }, stopCh) + }) if werr != nil { return nil, nil, werr } diff --git a/cmd/yurthub/app/config/config.go b/cmd/yurthub/app/config/config.go index 69d8dd8b891..a577755d331 100644 --- a/cmd/yurthub/app/config/config.go +++ b/cmd/yurthub/app/config/config.go @@ -17,6 +17,7 @@ limitations under the License. package config import ( + "context" "fmt" "net" "net/url" @@ -178,7 +179,7 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) { return nil, err } certMgr.Start() - err = wait.PollImmediate(5*time.Second, 4*time.Minute, func() (bool, error) { + err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 4*time.Minute, true, func(ctx context.Context) (bool, error) { isReady := certMgr.Ready() if isReady { return true, nil diff --git a/cmd/yurthub/app/start.go b/cmd/yurthub/app/start.go index 7738fdd2268..37dc86299a0 100644 --- a/cmd/yurthub/app/start.go +++ b/cmd/yurthub/app/start.go @@ -29,6 +29,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" + "k8s.io/component-base/cli/globalflag" "k8s.io/klog/v2" "github.com/openyurtio/openyurt/cmd/yurthub/app/config" @@ -84,6 +85,7 @@ func NewCmdStartYurtHub(ctx context.Context) *cobra.Command { }, } + globalflag.AddGlobalFlags(cmd.Flags(), cmd.Name()) yurtHubOptions.AddFlags(cmd.Flags()) return cmd } @@ -350,7 +352,7 @@ func coordinatorRun(ctx context.Context, } func yurtCoordinatorTransportMgrGetter(coordinatorCertMgr *coordinatorcertmgr.CertManager, stopCh <-chan struct{}) (transport.Interface, error) { - err := wait.PollImmediate(5*time.Second, 4*time.Minute, func() (done bool, err error) { + err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 4*time.Minute, true, func(ctx context.Context) (done bool, err error) { klog.Info("waiting for preparing certificates for coordinator client and node lease proxy client") if coordinatorCertMgr.GetAPIServerClientCert() == nil { return false, nil diff --git a/go.mod b/go.mod index c6f9aa29f88..e91b29723a7 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/openyurtio/openyurt -go 1.18 +go 1.20 require ( github.com/aliyun/alibaba-cloud-sdk-go v1.62.156 @@ -16,202 +16,196 @@ require ( github.com/hashicorp/go-version v1.6.0 github.com/jarcoal/httpmock v1.3.0 github.com/lithammer/dedent v1.1.0 - github.com/onsi/ginkgo/v2 v2.1.4 - github.com/onsi/gomega v1.19.0 + github.com/onsi/ginkgo/v2 v2.11.0 + github.com/onsi/gomega v1.27.10 github.com/opencontainers/selinux v1.11.0 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 github.com/projectcalico/api v0.0.0-20230222223746-44aa60c2201f - github.com/prometheus/client_golang v1.15.1 + github.com/prometheus/client_golang v1.16.0 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 github.com/vishvananda/netlink v1.2.1-beta.2 - go.etcd.io/etcd/api/v3 v3.5.0 - go.etcd.io/etcd/client/pkg/v3 v3.5.0 - go.etcd.io/etcd/client/v3 v3.5.0 - golang.org/x/net v0.22.0 + go.etcd.io/etcd/api/v3 v3.5.9 + go.etcd.io/etcd/client/pkg/v3 v3.5.9 + go.etcd.io/etcd/client/v3 v3.5.9 + golang.org/x/net v0.23.0 golang.org/x/sys v0.18.0 google.golang.org/grpc v1.57.1 gopkg.in/cheggaaa/pb.v1 v1.0.28 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.22.3 - k8s.io/apiextensions-apiserver v0.22.2 - k8s.io/apimachinery v0.22.3 - k8s.io/apiserver v0.22.3 - k8s.io/cli-runtime v0.22.3 - k8s.io/client-go v0.22.3 - k8s.io/cluster-bootstrap v0.22.3 - k8s.io/component-base v0.22.3 - k8s.io/component-helpers v0.22.3 - k8s.io/controller-manager v0.22.3 - k8s.io/klog/v2 v2.9.0 - k8s.io/kube-controller-manager v0.0.0 - k8s.io/kubectl v0.22.3 - k8s.io/kubelet v0.0.0 - k8s.io/kubernetes v1.22.3 - k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b + k8s.io/api v0.28.9 + k8s.io/apiextensions-apiserver v0.28.9 + k8s.io/apimachinery v0.28.9 + k8s.io/apiserver v0.28.9 + k8s.io/cli-runtime v0.28.9 + k8s.io/client-go v0.28.9 + k8s.io/cluster-bootstrap v0.28.9 + k8s.io/component-base v0.28.9 + k8s.io/component-helpers v0.28.9 + k8s.io/controller-manager v0.28.9 + k8s.io/klog/v2 v2.100.1 + k8s.io/kube-controller-manager v0.28.9 + k8s.io/kubectl v0.28.9 + k8s.io/kubelet v0.28.9 + k8s.io/kubernetes v0.0.0-00010101000000-000000000000 + k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 sigs.k8s.io/apiserver-network-proxy v0.0.15 - sigs.k8s.io/controller-runtime v0.10.3 + sigs.k8s.io/controller-runtime v0.16.5 ) require ( - github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd // indirect + github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect - github.com/go-errors/errors v1.0.1 // indirect + github.com/go-errors/errors v1.4.2 // indirect github.com/google/btree v1.0.1 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect - github.com/mitchellh/go-wordwrap v1.0.0 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect - github.com/russross/blackfriday v1.5.2 // indirect - github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect - go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect - sigs.k8s.io/kustomize/api v0.8.11 // indirect - sigs.k8s.io/kustomize/kyaml v0.11.0 // indirect + github.com/xlab/treeprint v1.2.0 // indirect + go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect + sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect + sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect ) require ( - cloud.google.com/go/compute v1.19.1 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/BurntSushi/toml v0.4.1 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect - github.com/PuerkitoBio/purell v1.1.1 // indirect - github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect + github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/blang/semver v3.5.1+incompatible // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect - github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd/v22 v22.3.2 // indirect + github.com/chai2010/gettext-go v1.0.2 // indirect + github.com/coreos/go-semver v0.3.1 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/cyphar/filepath-securejoin v0.2.4 // indirect - github.com/docker/distribution v2.7.1+incompatible // indirect - github.com/emicklei/go-restful v2.16.0+incompatible // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fatih/camelcase v1.0.0 // indirect - github.com/felixge/httpsnoop v1.0.1 // indirect + github.com/fatih/color v1.16.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/fvbommel/sortorder v1.0.1 // indirect + github.com/fvbommel/sortorder v1.1.0 // indirect github.com/fxamacker/cbor/v2 v2.4.0 // indirect - github.com/go-logr/logr v0.4.0 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.19.5 // indirect - github.com/go-openapi/swag v0.19.14 // indirect + github.com/go-logr/logr v1.2.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.3 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.13.0 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/cel-go v0.16.1 // indirect + github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/googleapis/gnostic v0.5.5 // indirect + github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect - github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/leodido/go-urn v1.2.3 // indirect - github.com/mailru/easyjson v0.7.6 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-runewidth v0.0.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/moby/spdystream v0.2.0 // indirect - github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect + github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/runc v1.1.12 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect - github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/spf13/afero v1.6.0 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect github.com/stretchr/objx v0.5.0 // indirect - github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect + github.com/vishvananda/netns v0.0.4 // indirect github.com/x448/float16 v0.8.4 // indirect - go.opentelemetry.io/contrib v0.20.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 // indirect - go.opentelemetry.io/otel v0.20.0 // indirect - go.opentelemetry.io/otel/exporters/otlp v0.20.0 // indirect - go.opentelemetry.io/otel/metric v0.20.0 // indirect - go.opentelemetry.io/otel/sdk v0.20.0 // indirect - go.opentelemetry.io/otel/sdk/export/metric v0.20.0 // indirect - go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect - go.opentelemetry.io/otel/trace v0.20.0 // indirect - go.opentelemetry.io/proto/otlp v0.7.0 // indirect - go.uber.org/atomic v1.9.0 // indirect - go.uber.org/multierr v1.6.0 // indirect - go.uber.org/zap v1.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.1 // indirect + go.opentelemetry.io/otel v1.10.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 // indirect + go.opentelemetry.io/otel/metric v0.31.0 // indirect + go.opentelemetry.io/otel/sdk v1.10.0 // indirect + go.opentelemetry.io/otel/trace v1.10.0 // indirect + go.opentelemetry.io/proto/otlp v0.19.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.25.0 // indirect golang.org/x/crypto v0.21.0 // indirect - golang.org/x/oauth2 v0.7.0 // indirect - golang.org/x/sync v0.3.0 // indirect + golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 // indirect + golang.org/x/oauth2 v0.8.0 // indirect + golang.org/x/sync v0.5.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect - gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect + golang.org/x/tools v0.16.1 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.66.2 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/cloud-provider v0.22.3 // indirect - k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect + k8s.io/cloud-provider v0.28.9 // indirect + k8s.io/kms v0.28.9 // indirect + k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) replace ( - k8s.io/api => k8s.io/api v0.22.3 - k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.22.3 - k8s.io/apimachinery => k8s.io/apimachinery v0.22.3 - k8s.io/apiserver => k8s.io/apiserver v0.22.3 - k8s.io/cli-runtime => k8s.io/cli-runtime v0.22.3 - k8s.io/client-go => k8s.io/client-go v0.22.3 - k8s.io/cloud-provider => k8s.io/cloud-provider v0.22.3 - k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.22.3 - k8s.io/code-generator => k8s.io/code-generator v0.22.3 - k8s.io/component-base => k8s.io/component-base v0.22.3 - k8s.io/component-helpers => k8s.io/component-helpers v0.22.3 - k8s.io/controller-manager => k8s.io/controller-manager v0.22.3 - k8s.io/cri-api => k8s.io/cri-api v0.22.3 - k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.22.3 - k8s.io/klog/v2 => k8s.io/klog/v2 v2.9.0 - k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.22.3 - k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.22.3 - k8s.io/kube-proxy => k8s.io/kube-proxy v0.22.3 - k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.22.3 - k8s.io/kubectl => k8s.io/kubectl v0.22.3 - k8s.io/kubelet => k8s.io/kubelet v0.22.3 - k8s.io/kubernetes => github.com/kubernetes/kubernetes v1.22.3 - k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.22.3 - k8s.io/metrics => k8s.io/metrics v0.22.3 - k8s.io/mount-utils => k8s.io/mount-utils v0.22.3 - k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.22.3 - k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.22.3 + k8s.io/api => k8s.io/api v0.28.9 + k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.28.9 + k8s.io/apimachinery => k8s.io/apimachinery v0.28.9 + k8s.io/apiserver => k8s.io/apiserver v0.28.9 + k8s.io/cli-runtime => k8s.io/cli-runtime v0.28.9 + k8s.io/client-go => k8s.io/client-go v0.28.9 + k8s.io/cloud-provider => k8s.io/cloud-provider v0.28.9 + k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.28.9 + k8s.io/code-generator => k8s.io/code-generator v0.28.9 + k8s.io/component-base => k8s.io/component-base v0.28.9 + k8s.io/component-helpers => k8s.io/component-helpers v0.28.9 + k8s.io/controller-manager => k8s.io/controller-manager v0.28.9 + k8s.io/cri-api => k8s.io/cri-api v0.28.9 + k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.28.9 + k8s.io/klog/v2 => k8s.io/klog/v2 v2.100.1 + k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.28.9 + k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.28.9 + k8s.io/kube-proxy => k8s.io/kube-proxy v0.28.9 + k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.28.9 + k8s.io/kubectl => k8s.io/kubectl v0.28.9 + k8s.io/kubelet => k8s.io/kubelet v0.28.9 + k8s.io/kubernetes => github.com/kubernetes/kubernetes v1.28.9 + k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.28.9 + k8s.io/metrics => k8s.io/metrics v0.28.9 + k8s.io/mount-utils => k8s.io/mount-utils v0.28.9 + k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.28.9 + k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.28.9 sigs.k8s.io/apiserver-network-proxy => github.com/openyurtio/apiserver-network-proxy v0.1.0 - sigs.k8s.io/apiserver-network-proxy/konnectivity-client => sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22 -) - -exclude ( - k8s.io/api v0.23.2 - k8s.io/apimachinery v0.23.2 - k8s.io/apiserver v0.23.2 - k8s.io/client-go v0.23.2 - k8s.io/component-base v0.23.2 + sigs.k8s.io/apiserver-network-proxy/konnectivity-client => sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.2 ) diff --git a/go.sum b/go.sum index 3b1cd40cedc..1a532381c73 100644 --- a/go.sum +++ b/go.sum @@ -1,225 +1,755 @@ -bitbucket.org/bertimus9/systemstat v0.0.0-20180207000608-0eeff89b0690/go.mod h1:Ulb78X89vxKYgdL24HMTiXYHlyHEvruOj1ZPlqeNEZM= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U= +cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= +cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= +cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= +cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= +cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= +cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= +cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= +cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= +cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= +cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE= +cloud.google.com/go/accesscontextmanager v1.6.0/go.mod h1:8XCvZWfYw3K/ji0iVnp+6pu7huxoQTLmxAbVjbloTtM= +cloud.google.com/go/accesscontextmanager v1.7.0/go.mod h1:CEGLewx8dwa33aDAZQujl7Dx+uYhS0eay198wB/VumQ= +cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= +cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= +cloud.google.com/go/aiplatform v1.27.0/go.mod h1:Bvxqtl40l0WImSb04d0hXFU7gDOiq9jQmorivIiWcKg= +cloud.google.com/go/aiplatform v1.35.0/go.mod h1:7MFT/vCaOyZT/4IIFfxH4ErVg/4ku6lKv3w0+tFTgXQ= +cloud.google.com/go/aiplatform v1.36.1/go.mod h1:WTm12vJRPARNvJ+v6P52RDHCNe4AhvjcIZ/9/RRHy/k= +cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw= +cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= +cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= +cloud.google.com/go/analytics v0.17.0/go.mod h1:WXFa3WSym4IZ+JiKmavYdJwGG/CvpqiqczmL59bTD9M= +cloud.google.com/go/analytics v0.18.0/go.mod h1:ZkeHGQlcIPkw0R/GW+boWHhCOR43xz9RN/jn7WcqfIE= +cloud.google.com/go/analytics v0.19.0/go.mod h1:k8liqf5/HCnOUkbawNtrWWc+UAzyDlW89doe8TtoDsE= +cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= +cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= +cloud.google.com/go/apigateway v1.5.0/go.mod h1:GpnZR3Q4rR7LVu5951qfXPJCHquZt02jf7xQx7kpqN8= +cloud.google.com/go/apigeeconnect v1.3.0/go.mod h1:G/AwXFAKo0gIXkPTVfZDd2qA1TxBXJ3MgMRBQkIi9jc= +cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn/pkS3QlHp0Ni04= +cloud.google.com/go/apigeeconnect v1.5.0/go.mod h1:KFaCqvBRU6idyhSNyn3vlHXc8VMDJdRmwDF6JyFRqZ8= +cloud.google.com/go/apigeeregistry v0.4.0/go.mod h1:EUG4PGcsZvxOXAdyEghIdXwAEi/4MEaoqLMLDMIwKXY= +cloud.google.com/go/apigeeregistry v0.5.0/go.mod h1:YR5+s0BVNZfVOUkMa5pAR2xGd0A473vA5M7j247o1wM= +cloud.google.com/go/apigeeregistry v0.6.0/go.mod h1:BFNzW7yQVLZ3yj0TKcwzb8n25CFBri51GVGOEUcgQsc= +cloud.google.com/go/apikeys v0.4.0/go.mod h1:XATS/yqZbaBK0HOssf+ALHp8jAlNHUgyfprvNcBIszU= +cloud.google.com/go/apikeys v0.5.0/go.mod h1:5aQfwY4D+ewMMWScd3hm2en3hCj+BROlyrt3ytS7KLI= +cloud.google.com/go/apikeys v0.6.0/go.mod h1:kbpXu5upyiAlGkKrJgQl8A0rKNNJ7dQ377pdroRSSi8= +cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno= +cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak= +cloud.google.com/go/appengine v1.6.0/go.mod h1:hg6i0J/BD2cKmDJbaFSYHFyZkgBEfQrDg/X0V5fJn84= +cloud.google.com/go/appengine v1.7.0/go.mod h1:eZqpbHFCqRGa2aCdope7eC0SWLV1j0neb/QnMJVWx6A= +cloud.google.com/go/appengine v1.7.1/go.mod h1:IHLToyb/3fKutRysUlFO0BPt5j7RiQ45nrzEJmKTo6E= +cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= +cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= +cloud.google.com/go/area120 v0.7.0/go.mod h1:a3+8EUD1SX5RUcCs3MY5YasiO1z6yLiNLRiFrykbynY= +cloud.google.com/go/area120 v0.7.1/go.mod h1:j84i4E1RboTWjKtZVWXPqvK5VHQFJRF2c1Nm69pWm9k= +cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= +cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= +cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0= +cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc= +cloud.google.com/go/artifactregistry v1.11.1/go.mod h1:lLYghw+Itq9SONbCa1YWBoWs1nOucMH0pwXN1rOBZFI= +cloud.google.com/go/artifactregistry v1.11.2/go.mod h1:nLZns771ZGAwVLzTX/7Al6R9ehma4WUEhZGWV6CeQNQ= +cloud.google.com/go/artifactregistry v1.12.0/go.mod h1:o6P3MIvtzTOnmvGagO9v/rOjjA0HmhJ+/6KAXrmYDCI= +cloud.google.com/go/artifactregistry v1.13.0/go.mod h1:uy/LNfoOIivepGhooAUpL1i30Hgee3Cu0l4VTWHUC08= +cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= +cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= +cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= +cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ= +cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY= +cloud.google.com/go/asset v1.11.1/go.mod h1:fSwLhbRvC9p9CXQHJ3BgFeQNM4c9x10lqlrdEUYXlJo= +cloud.google.com/go/asset v1.12.0/go.mod h1:h9/sFOa4eDIyKmH6QMpm4eUK3pDojWnUhTgJlk762Hg= +cloud.google.com/go/asset v1.13.0/go.mod h1:WQAMyYek/b7NBpYq/K4KJWcRqzoalEsxz/t/dTk4THw= +cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= +cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= +cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= +cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= +cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= +cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E= +cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= +cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= +cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= +cloud.google.com/go/automl v1.8.0/go.mod h1:xWx7G/aPEe/NP+qzYXktoBSDfjO+vnKMGgsApGJJquM= +cloud.google.com/go/automl v1.12.0/go.mod h1:tWDcHDp86aMIuHmyvjuKeeHEGq76lD7ZqfGLN6B0NuU= +cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc= +cloud.google.com/go/baremetalsolution v0.4.0/go.mod h1:BymplhAadOO/eBa7KewQ0Ppg4A4Wplbn+PsFKRLo0uI= +cloud.google.com/go/baremetalsolution v0.5.0/go.mod h1:dXGxEkmR9BMwxhzBhV0AioD0ULBmuLZI8CdwalUxuss= +cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE= +cloud.google.com/go/batch v0.4.0/go.mod h1:WZkHnP43R/QCGQsZ+0JyG4i79ranE2u8xvjq/9+STPE= +cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oevL159g= +cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= +cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= +cloud.google.com/go/beyondcorp v0.4.0/go.mod h1:3ApA0mbhHx6YImmuubf5pyW8srKnCEPON32/5hj+RmM= +cloud.google.com/go/beyondcorp v0.5.0/go.mod h1:uFqj9X+dSfrheVp7ssLTaRHd2EHqSL4QZmH4e8WXGGU= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= +cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= +cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc= +cloud.google.com/go/bigquery v1.47.0/go.mod h1:sA9XOgy0A8vQK9+MWhEQTY6Tix87M/ZurWFIxmF9I/E= +cloud.google.com/go/bigquery v1.48.0/go.mod h1:QAwSz+ipNgfL5jxiaK7weyOhzdoAy1zFm0Nf1fysJac= +cloud.google.com/go/bigquery v1.49.0/go.mod h1:Sv8hMmTFFYBlt/ftw2uN6dFdQPzBlREY9yBh7Oy7/4Q= +cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= +cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= +cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= +cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= +cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y= +cloud.google.com/go/billing v1.12.0/go.mod h1:yKrZio/eu+okO/2McZEbch17O5CB5NpZhhXG6Z766ss= +cloud.google.com/go/billing v1.13.0/go.mod h1:7kB2W9Xf98hP9Sr12KfECgfGclsH3CQR0R08tnRlRbc= +cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= +cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= +cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= +cloud.google.com/go/binaryauthorization v1.4.0/go.mod h1:tsSPQrBd77VLplV70GUhBf/Zm3FsKmgSqgm4UmiDItk= +cloud.google.com/go/binaryauthorization v1.5.0/go.mod h1:OSe4OU1nN/VswXKRBmciKpo9LulY41gch5c68htf3/Q= +cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg= +cloud.google.com/go/certificatemanager v1.4.0/go.mod h1:vowpercVFyqs8ABSmrdV+GiFf2H/ch3KyudYQEMM590= +cloud.google.com/go/certificatemanager v1.6.0/go.mod h1:3Hh64rCKjRAX8dXgRAyOcY5vQ/fE1sh8o+Mdd6KPgY8= +cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk= +cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL8yBMbKUk= +cloud.google.com/go/channel v1.11.0/go.mod h1:IdtI0uWGqhEeatSB62VOoJ8FSUhJ9/+iGkJVqp74CGE= +cloud.google.com/go/channel v1.12.0/go.mod h1:VkxCGKASi4Cq7TbXxlaBezonAYpp1GCnKMY6tnMQnLU= +cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= +cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= +cloud.google.com/go/cloudbuild v1.6.0/go.mod h1:UIbc/w9QCbH12xX+ezUsgblrWv+Cv4Tw83GiSMHOn9M= +cloud.google.com/go/cloudbuild v1.7.0/go.mod h1:zb5tWh2XI6lR9zQmsm1VRA+7OCuve5d8S+zJUul8KTg= +cloud.google.com/go/cloudbuild v1.9.0/go.mod h1:qK1d7s4QlO0VwfYn5YuClDGg2hfmLZEb4wQGAbIgL1s= +cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= +cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= +cloud.google.com/go/clouddms v1.5.0/go.mod h1:QSxQnhikCLUw13iAbffF2CZxAER3xDGNHjsTAkQJcQA= +cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= +cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= +cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= +cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI= +cloud.google.com/go/cloudtasks v1.9.0/go.mod h1:w+EyLsVkLWHcOaqNEyvcKAsWp9p29dL6uL9Nst1cI7Y= +cloud.google.com/go/cloudtasks v1.10.0/go.mod h1:NDSoTLkZ3+vExFEWu2UJV1arUyzVDAiZtdWcsUyNwBs= +cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= +cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= +cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= +cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE= +cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= +cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= +cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= +cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= +cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= +cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= +cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= +cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= +cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= +cloud.google.com/go/container v1.13.1/go.mod h1:6wgbMPeQRw9rSnKBCAJXnds3Pzj03C4JHamr8asWKy4= +cloud.google.com/go/container v1.14.0/go.mod h1:3AoJMPhHfLDxLvrlVWaK57IXzaPnLaZq63WX59aQBfM= +cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA= +cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= +cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= +cloud.google.com/go/containeranalysis v0.7.0/go.mod h1:9aUL+/vZ55P2CXfuZjS4UjQ9AgXoSw8Ts6lemfmxBxI= +cloud.google.com/go/containeranalysis v0.9.0/go.mod h1:orbOANbwk5Ejoom+s+DUCTTJ7IBdBQJDcSylAx/on9s= +cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= +cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= +cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= +cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H0/tc8f8ZbZIE= +cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOXxZoKYF5wdISM= +cloud.google.com/go/datacatalog v1.8.1/go.mod h1:RJ58z4rMp3gvETA465Vg+ag8BGgBdnRPEMMSTr5Uv+M= +cloud.google.com/go/datacatalog v1.12.0/go.mod h1:CWae8rFkfp6LzLumKOnmVh4+Zle4A3NXLzVJ1d1mRm0= +cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= +cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= +cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= +cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE= +cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= +cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= +cloud.google.com/go/dataform v0.5.0/go.mod h1:GFUYRe8IBa2hcomWplodVmUx/iTL0FrsauObOM3Ipr0= +cloud.google.com/go/dataform v0.6.0/go.mod h1:QPflImQy33e29VuapFdf19oPbE4aYTJxr31OAPV+ulA= +cloud.google.com/go/dataform v0.7.0/go.mod h1:7NulqnVozfHvWUBpMDfKMUESr+85aJsC/2O0o3jWPDE= +cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38= +cloud.google.com/go/datafusion v1.5.0/go.mod h1:Kz+l1FGHB0J+4XF2fud96WMmRiq/wj8N9u007vyXZ2w= +cloud.google.com/go/datafusion v1.6.0/go.mod h1:WBsMF8F1RhSXvVM8rCV3AeyWVxcC2xY6vith3iw3S+8= +cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= +cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= +cloud.google.com/go/datalabeling v0.7.0/go.mod h1:WPQb1y08RJbmpM3ww0CSUAGweL0SxByuW2E+FU+wXcM= +cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA= +cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0kZCCKsU0A= +cloud.google.com/go/dataplex v1.5.2/go.mod h1:cVMgQHsmfRoI5KFYq4JtIBEUbYwc3c7tXmIDhRmNNVQ= +cloud.google.com/go/dataplex v1.6.0/go.mod h1:bMsomC/aEJOSpHXdFKFGQ1b0TDPIeL28nJObeO1ppRs= +cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= +cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= +cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= +cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= +cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= +cloud.google.com/go/dataqna v0.7.0/go.mod h1:Lx9OcIIeqCrw1a6KdO3/5KMP1wAmTc0slZWwP12Qq3c= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= +cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= +cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= +cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= +cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= +cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2/XoS5yi88q4= +cloud.google.com/go/datastream v1.6.0/go.mod h1:6LQSuswqLa7S4rPAOZFVjHIG3wJIjZcZrw8JDEDJuIs= +cloud.google.com/go/datastream v1.7.0/go.mod h1:uxVRMm2elUSPuh65IbZpzJNMbuzkcvu5CjMqVIUHrww= +cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= +cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= +cloud.google.com/go/deploy v1.6.0/go.mod h1:f9PTHehG/DjCom3QH0cntOVRm93uGBDt2vKzAPwpXQI= +cloud.google.com/go/deploy v1.8.0/go.mod h1:z3myEJnA/2wnB4sgjqdMfgxCA0EqC3RBTNcVPs93mtQ= +cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= +cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= +cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= +cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz0njv7sMx/iek= +cloud.google.com/go/dialogflow v1.19.0/go.mod h1:JVmlG1TwykZDtxtTXujec4tQ+D8SBFMoosgy+6Gn0s0= +cloud.google.com/go/dialogflow v1.29.0/go.mod h1:b+2bzMe+k1s9V+F2jbJwpHPzrnIyHihAdRFMtn2WXuM= +cloud.google.com/go/dialogflow v1.31.0/go.mod h1:cuoUccuL1Z+HADhyIA7dci3N5zUssgpBJmCzI6fNRB4= +cloud.google.com/go/dialogflow v1.32.0/go.mod h1:jG9TRJl8CKrDhMEcvfcfFkkpp8ZhgPz3sBGmAUYJ2qE= +cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= +cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= +cloud.google.com/go/dlp v1.9.0/go.mod h1:qdgmqgTyReTz5/YNSSuueR8pl7hO0o9bQ39ZhtgkWp4= +cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= +cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= +cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k= +cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc0ZvxxfQY1bg4= +cloud.google.com/go/documentai v1.16.0/go.mod h1:o0o0DLTEZ+YnJZ+J4wNfTxmDVyrkzFvttBXXtYRMHkM= +cloud.google.com/go/documentai v1.18.0/go.mod h1:F6CK6iUH8J81FehpskRmhLq/3VlwQvb7TvwOceQ2tbs= +cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= +cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= +cloud.google.com/go/domains v0.8.0/go.mod h1:M9i3MMDzGFXsydri9/vW+EWz9sWb4I6WyHqdlAk0idE= +cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= +cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= +cloud.google.com/go/edgecontainer v0.3.0/go.mod h1:FLDpP4nykgwwIfcLt6zInhprzw0lEi2P1fjO6Ie0qbc= +cloud.google.com/go/edgecontainer v1.0.0/go.mod h1:cttArqZpBB2q58W/upSG++ooo6EsblxDIolxa3jSjbY= +cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= +cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= +cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8= +cloud.google.com/go/essentialcontacts v1.5.0/go.mod h1:ay29Z4zODTuwliK7SnX8E86aUF2CTzdNtvv42niCX0M= +cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= +cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw= +cloud.google.com/go/eventarc v1.10.0/go.mod h1:u3R35tmZ9HvswGRBnF48IlYgYeBcPUCjkr4BTdem2Kw= +cloud.google.com/go/eventarc v1.11.0/go.mod h1:PyUjsUKPWoRBCHeOxZd/lbOOjahV41icXyUY5kSTvVY= +cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= +cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= +cloud.google.com/go/filestore v1.5.0/go.mod h1:FqBXDWBp4YLHqRnVGveOkHDf8svj9r5+mUDLupOWEDs= +cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466dre85Kydllg= +cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= +cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= +cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= +cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= +cloud.google.com/go/functions v1.9.0/go.mod h1:Y+Dz8yGguzO3PpIjhLTbnqV1CWmgQ5UwtlpzoyquQ08= +cloud.google.com/go/functions v1.10.0/go.mod h1:0D3hEOe3DbEvCXtYOZHQZmD+SzYsi1YbI7dGvHfldXw= +cloud.google.com/go/functions v1.12.0/go.mod h1:AXWGrF3e2C/5ehvwYo/GH6O5s09tOPksiKhz+hH8WkA= +cloud.google.com/go/functions v1.13.0/go.mod h1:EU4O007sQm6Ef/PwRsI8N2umygGqPBS/IZQKBQBcJ3c= +cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= +cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= +cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w= +cloud.google.com/go/gaming v1.8.0/go.mod h1:xAqjS8b7jAVW0KFYeRUxngo9My3f33kFmua++Pi+ggM= +cloud.google.com/go/gaming v1.9.0/go.mod h1:Fc7kEmCObylSWLO334NcO+O9QMDyz+TKC4v1D7X+Bc0= +cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60= +cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2HDlj4TsBRAo= +cloud.google.com/go/gkebackup v0.4.0/go.mod h1:byAyBGUwYGEEww7xsbnUTBHIYcOPy/PgUWUtOeRm9Vg= +cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= +cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= +cloud.google.com/go/gkeconnect v0.7.0/go.mod h1:SNfmVqPkaEi3bF/B3CNZOAYPYdg7sU+obZ+QTky2Myw= +cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= +cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= +cloud.google.com/go/gkehub v0.11.0/go.mod h1:JOWHlmN+GHyIbuWQPl47/C2RFhnFKH38jH9Ascu3n0E= +cloud.google.com/go/gkehub v0.12.0/go.mod h1:djiIwwzTTBrF5NaXCGv3mf7klpEMcST17VBTVVDcuaw= +cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA= +cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= +cloud.google.com/go/gkemulticloud v0.5.0/go.mod h1:W0JDkiyi3Tqh0TJr//y19wyb1yf8llHVto2Htf2Ja3Y= +cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= +cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= +cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o= +cloud.google.com/go/gsuiteaddons v1.5.0/go.mod h1:TFCClYLd64Eaa12sFVmUyG62tk4mdIsI7pAnSXRkcFo= +cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= +cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= +cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= +cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= +cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= +cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= +cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= +cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= +cloud.google.com/go/iap v1.7.0/go.mod h1:beqQx56T9O1G1yNPph+spKpNibDlYIiIixiqsQXxLIo= +cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74= +cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= +cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= +cloud.google.com/go/ids v1.3.0/go.mod h1:JBdTYwANikFKaDP6LtW5JAi4gubs57SVNQjemdt6xV4= +cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= +cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g= +cloud.google.com/go/iot v1.5.0/go.mod h1:mpz5259PDl3XJthEmh9+ap0affn/MqNSP4My77Qql9o= +cloud.google.com/go/iot v1.6.0/go.mod h1:IqdAsmE2cTYYNO1Fvjfzo9po179rAtJeVGUvkLN3rLE= +cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= +cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= +cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= +cloud.google.com/go/kms v1.8.0/go.mod h1:4xFEhYFqvW+4VMELtZyxomGSYtSQKzM178ylFW4jMAg= +cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w= +cloud.google.com/go/kms v1.10.0/go.mod h1:ng3KTUtQQU9bPX3+QGLsflZIHlkbn8amFAMY63m8d24= +cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI= +cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= +cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= +cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= +cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8= +cloud.google.com/go/language v1.9.0/go.mod h1:Ns15WooPM5Ad/5no/0n81yUetis74g3zrbeJBE+ptUY= +cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= +cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= +cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo= +cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= +cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= +cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= +cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= +cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= +cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= +cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= +cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA= +cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI= +cloud.google.com/go/maps v0.6.0/go.mod h1:o6DAMMfb+aINHz/p/jbcY+mYeXBoZoxTfdSQ8VAJaCw= +cloud.google.com/go/maps v0.7.0/go.mod h1:3GnvVl3cqeSvgMcpRlQidXsPYuDGQ8naBis7MVzpXsY= +cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= +cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= +cloud.google.com/go/mediatranslation v0.7.0/go.mod h1:LCnB/gZr90ONOIQLgSXagp8XUW1ODs2UmUMvcgMfI2I= +cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= +cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= +cloud.google.com/go/memcache v1.6.0/go.mod h1:XS5xB0eQZdHtTuTF9Hf8eJkKtR3pVRCcvJwtm68T3rA= +cloud.google.com/go/memcache v1.7.0/go.mod h1:ywMKfjWhNtkQTxrWxCkCFkoPjLHPW6A7WOTVI8xy3LY= +cloud.google.com/go/memcache v1.9.0/go.mod h1:8oEyzXCu+zo9RzlEaEjHl4KkgjlNDaXbCQeQWlzNFJM= +cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= +cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= +cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSoxlKKDqBGFS8= +cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI= +cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJPWOodSaf45Eo= +cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= +cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= +cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w= +cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw= +cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= +cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= +cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= +cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8= +cloud.google.com/go/networkconnectivity v1.10.0/go.mod h1:UP4O4sWXJG13AqrTdQCD9TnLGEbtNRqjuaaA7bNjF5E= +cloud.google.com/go/networkconnectivity v1.11.0/go.mod h1:iWmDD4QF16VCDLXUqvyspJjIEtBR/4zq5hwnY2X3scM= +cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= +cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= +cloud.google.com/go/networkmanagement v1.6.0/go.mod h1:5pKPqyXjB/sgtvB5xqOemumoQNB7y95Q7S+4rjSOPYY= +cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= +cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= +cloud.google.com/go/networksecurity v0.7.0/go.mod h1:mAnzoxx/8TBSyXEeESMy9OOYwo1v+gZ5eMRnsT5bC8k= +cloud.google.com/go/networksecurity v0.8.0/go.mod h1:B78DkqsxFG5zRSVuwYFRZ9Xz8IcQ5iECsNrPn74hKHU= +cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= +cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= +cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA= +cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vuYs+kBJ/gu0= +cloud.google.com/go/notebooks v1.7.0/go.mod h1:PVlaDGfJgj1fl1S3dUwhFMXFgfYGhYQt2164xOMONmE= +cloud.google.com/go/notebooks v1.8.0/go.mod h1:Lq6dYKOYOWUCTvw5t2q1gp1lAp0zxAxRycayS0iJcqQ= +cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= +cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= +cloud.google.com/go/optimization v1.3.1/go.mod h1:IvUSefKiwd1a5p0RgHDbWCIbDFgKuEdB+fPPuP0IDLI= +cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA= +cloud.google.com/go/orchestration v1.4.0/go.mod h1:6W5NLFWs2TlniBphAViZEVhrXRSMgUGDfW7vrWKvsBk= +cloud.google.com/go/orchestration v1.6.0/go.mod h1:M62Bevp7pkxStDfFfTuCOaXgaaqRAga1yKyoMtEoWPQ= +cloud.google.com/go/orgpolicy v1.4.0/go.mod h1:xrSLIV4RePWmP9P3tBl8S93lTmlAxjm06NSm2UTmKvE= +cloud.google.com/go/orgpolicy v1.5.0/go.mod h1:hZEc5q3wzwXJaKrsx5+Ewg0u1LxJ51nNFlext7Tanwc= +cloud.google.com/go/orgpolicy v1.10.0/go.mod h1:w1fo8b7rRqlXlIJbVhOMPrwVljyuW5mqssvBtU18ONc= +cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= +cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= +cloud.google.com/go/osconfig v1.9.0/go.mod h1:Yx+IeIZJ3bdWmzbQU4fxNl8xsZ4amB+dygAwFPlvnNo= +cloud.google.com/go/osconfig v1.10.0/go.mod h1:uMhCzqC5I8zfD9zDEAfvgVhDS8oIjySWh+l4WK6GnWw= +cloud.google.com/go/osconfig v1.11.0/go.mod h1:aDICxrur2ogRd9zY5ytBLV89KEgT2MKB2L/n6x1ooPw= +cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= +cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= +cloud.google.com/go/oslogin v1.6.0/go.mod h1:zOJ1O3+dTU8WPlGEkFSh7qeHPPSoxrcMbbK1Nm2iX70= +cloud.google.com/go/oslogin v1.7.0/go.mod h1:e04SN0xO1UNJ1M5GP0vzVBFicIe4O53FOfcixIqTyXo= +cloud.google.com/go/oslogin v1.9.0/go.mod h1:HNavntnH8nzrn8JCTT5fj18FuJLFJc4NaZJtBnQtKFs= +cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= +cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= +cloud.google.com/go/phishingprotection v0.7.0/go.mod h1:8qJI4QKHoda/sb/7/YmMQ2omRLSLYSu9bU0EKCNI+Lk= +cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg= +cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE= +cloud.google.com/go/policytroubleshooter v1.5.0/go.mod h1:Rz1WfV+1oIpPdN2VvvuboLVRsB1Hclg3CKQ53j9l8vw= +cloud.google.com/go/policytroubleshooter v1.6.0/go.mod h1:zYqaPTsmfvpjm5ULxAyD/lINQxJ0DDsnWOP/GZ7xzBc= +cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= +cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= +cloud.google.com/go/privatecatalog v0.7.0/go.mod h1:2s5ssIFO69F5csTXcwBP7NPFTZvps26xGzvQ2PQaBYg= +cloud.google.com/go/privatecatalog v0.8.0/go.mod h1:nQ6pfaegeDAq/Q5lrfCQzQLhubPiZhSaNhIgfJlnIXs= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= +cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= +cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8= +cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4= +cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= +cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k= +cloud.google.com/go/pubsublite v1.7.0/go.mod h1:8hVMwRXfDfvGm3fahVbtDbiLePT3gpoiJYJY+vxWxVM= +cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= +cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= +cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= +cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= +cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE= +cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= +cloud.google.com/go/recaptchaenterprise/v2 v2.6.0/go.mod h1:RPauz9jeLtB3JVzg6nCbe12qNoaa8pXc4d/YukAmcnA= +cloud.google.com/go/recaptchaenterprise/v2 v2.7.0/go.mod h1:19wVj/fs5RtYtynAPJdDTb69oW0vNHYDBTbB4NvMD9c= +cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= +cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= +cloud.google.com/go/recommendationengine v0.7.0/go.mod h1:1reUcE3GIu6MeBz/h5xZJqNLuuVjNg1lmWMPyjatzac= +cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= +cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= +cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclClff6lhFVe9Bs= +cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70= +cloud.google.com/go/recommender v1.9.0/go.mod h1:PnSsnZY7q+VL1uax2JWkt/UegHssxjUVVCrX52CuEmQ= +cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= +cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= +cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA= +cloud.google.com/go/redis v1.10.0/go.mod h1:ThJf3mMBQtW18JzGgh41/Wld6vnDDc/F/F35UolRZPM= +cloud.google.com/go/redis v1.11.0/go.mod h1:/X6eicana+BWcUda5PpwZC48o37SiFVTFSs0fWAJ7uQ= +cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA= +cloud.google.com/go/resourcemanager v1.4.0/go.mod h1:MwxuzkumyTX7/a3n37gmsT3py7LIXwrShilPh3P1tR0= +cloud.google.com/go/resourcemanager v1.5.0/go.mod h1:eQoXNAiAvCf5PXxWxXjhKQoTMaUSNrEfg+6qdf/wots= +cloud.google.com/go/resourcemanager v1.6.0/go.mod h1:YcpXGRs8fDzcUl1Xw8uOVmI8JEadvhRIkoXXUNVYcVo= +cloud.google.com/go/resourcemanager v1.7.0/go.mod h1:HlD3m6+bwhzj9XCouqmeiGuni95NTrExfhoSrkC/3EI= +cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU= +cloud.google.com/go/resourcesettings v1.4.0/go.mod h1:ldiH9IJpcrlC3VSuCGvjR5of/ezRrOxFtpJoJo5SmXg= +cloud.google.com/go/resourcesettings v1.5.0/go.mod h1:+xJF7QSG6undsQDfsCJyqWXyBwUoJLhetkRMDRnIoXA= +cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= +cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= +cloud.google.com/go/retail v1.10.0/go.mod h1:2gDk9HsL4HMS4oZwz6daui2/jmKvqShXKQuB2RZ+cCc= +cloud.google.com/go/retail v1.11.0/go.mod h1:MBLk1NaWPmh6iVFSz9MeKG/Psyd7TAgm6y/9L2B4x9Y= +cloud.google.com/go/retail v1.12.0/go.mod h1:UMkelN/0Z8XvKymXFbD4EhFJlYKRx1FGhQkVPU5kF14= +cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do= +cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= +cloud.google.com/go/run v0.8.0/go.mod h1:VniEnuBwqjigv0A7ONfQUaEItaiCRVujlMqerPPiktM= +cloud.google.com/go/run v0.9.0/go.mod h1:Wwu+/vvg8Y+JUApMwEDfVfhetv30hCG4ZwDR/IXl2Qg= +cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= +cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= +cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= +cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44= +cloud.google.com/go/scheduler v1.8.0/go.mod h1:TCET+Y5Gp1YgHT8py4nlg2Sew8nUHMqcpousDgXJVQc= +cloud.google.com/go/scheduler v1.9.0/go.mod h1:yexg5t+KSmqu+njTIh3b7oYPheFtBWGcbVUYF1GGMIc= +cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= +cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= +cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= +cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU= +cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= +cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= +cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= +cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q= +cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA= +cloud.google.com/go/security v1.12.0/go.mod h1:rV6EhrpbNHrrxqlvW0BWAIawFWq3X90SduMJdFwtLB8= +cloud.google.com/go/security v1.13.0/go.mod h1:Q1Nvxl1PAgmeW0y3HTt54JYIvUdtcpYKVfIB8AOMZ+0= +cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= +cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= +cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk= +cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk= +cloud.google.com/go/securitycenter v1.18.1/go.mod h1:0/25gAzCM/9OL9vVx4ChPeM/+DlfGQJDwBy/UC8AKK0= +cloud.google.com/go/securitycenter v1.19.0/go.mod h1:LVLmSg8ZkkyaNy4u7HCIshAngSQ8EcIRREP3xBnyfag= +cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU= +cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s= +cloud.google.com/go/servicecontrol v1.10.0/go.mod h1:pQvyvSRh7YzUF2efw7H87V92mxU8FnFDawMClGCNuAA= +cloud.google.com/go/servicecontrol v1.11.0/go.mod h1:kFmTzYzTUIuZs0ycVqRHNaNhgR+UMUpw9n02l/pY+mc= +cloud.google.com/go/servicecontrol v1.11.1/go.mod h1:aSnNNlwEFBY+PWGQ2DoM0JJ/QUXqV5/ZD9DOLB7SnUk= +cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= +cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= +cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4= +cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UVXjkuw7q5XcG10wx1U= +cloud.google.com/go/servicedirectory v1.8.0/go.mod h1:srXodfhY1GFIPvltunswqXpVxFPpZjf8nkKQT7XcXaY= +cloud.google.com/go/servicedirectory v1.9.0/go.mod h1:29je5JjiygNYlmsGz8k6o+OZ8vd4f//bQLtvzkPPT/s= +cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= +cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= +cloud.google.com/go/servicemanagement v1.6.0/go.mod h1:aWns7EeeCOtGEX4OvZUWCCJONRZeFKiptqKf1D0l/Jc= +cloud.google.com/go/servicemanagement v1.8.0/go.mod h1:MSS2TDlIEQD/fzsSGfCdJItQveu9NXnUniTrq/L8LK4= +cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E= +cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= +cloud.google.com/go/serviceusage v1.5.0/go.mod h1:w8U1JvqUqwJNPEOTQjrMHkw3IaIFLoLsPLvsE3xueec= +cloud.google.com/go/serviceusage v1.6.0/go.mod h1:R5wwQcbOWsyuOfbP9tGdAnCAc6B9DRwPG1xtWMDeuPA= +cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= +cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= +cloud.google.com/go/shell v1.6.0/go.mod h1:oHO8QACS90luWgxP3N9iZVuEiSF84zNyLytb+qE2f9A= +cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos= +cloud.google.com/go/spanner v1.44.0/go.mod h1:G8XIgYdOK+Fbcpbs7p2fiprDw4CaZX63whnSMLVBxjk= +cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M= +cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= +cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= +cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= +cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco= +cloud.google.com/go/speech v1.14.1/go.mod h1:gEosVRPJ9waG7zqqnsHpYTOoAS4KouMRLDFMekpJ0J0= +cloud.google.com/go/speech v1.15.0/go.mod h1:y6oH7GhqCaZANH7+Oe0BhgIogsNInLlz542tg3VqeYI= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= +cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= +cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= +cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= +cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4= +cloud.google.com/go/storagetransfer v1.8.0/go.mod h1:JpegsHHU1eXg7lMHkvf+KE5XDJ7EQu0GwNJbbVGanEw= +cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= +cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= +cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM= +cloud.google.com/go/talent v1.4.0/go.mod h1:ezFtAgVuRf8jRsvyE6EwmbTK5LKciD4KVnHuDEFmOOA= +cloud.google.com/go/talent v1.5.0/go.mod h1:G+ODMj9bsasAEJkQSzO2uHQWXHHXUomArjWQQYkqK6c= +cloud.google.com/go/texttospeech v1.4.0/go.mod h1:FX8HQHA6sEpJ7rCMSfXuzBcysDAuWusNNNvN9FELDd8= +cloud.google.com/go/texttospeech v1.5.0/go.mod h1:oKPLhR4n4ZdQqWKURdwxMy0uiTS1xU161C8W57Wkea4= +cloud.google.com/go/texttospeech v1.6.0/go.mod h1:YmwmFT8pj1aBblQOI3TfKmwibnsfvhIBzPXcW4EBovc= +cloud.google.com/go/tpu v1.3.0/go.mod h1:aJIManG0o20tfDQlRIej44FcwGGl/cD0oiRyMKG19IQ= +cloud.google.com/go/tpu v1.4.0/go.mod h1:mjZaX8p0VBgllCzF6wcU2ovUXN9TONFLd7iz227X2Xg= +cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV61fPM= +cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28= +cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= +cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA= +cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk= +cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= +cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= +cloud.google.com/go/translate v1.5.0/go.mod h1:29YDSYveqqpA1CQFD7NQuP49xymq17RXNaUDdc0mNu0= +cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/translate v1.7.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= +cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= +cloud.google.com/go/video v1.12.0/go.mod h1:MLQew95eTuaNDEGriQdcYn0dTwf9oWiA4uYebxM5kdg= +cloud.google.com/go/video v1.13.0/go.mod h1:ulzkYlYgCp15N2AokzKjy7MQ9ejuynOJdf1tR5lGthk= +cloud.google.com/go/video v1.14.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= +cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= +cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= +cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= +cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= +cloud.google.com/go/videointelligence v1.9.0/go.mod h1:29lVRMPDYHikk3v8EdPSaL8Ku+eMzDljjuvRs105XoU= +cloud.google.com/go/videointelligence v1.10.0/go.mod h1:LHZngX1liVtUhZvi2uNS0VQuOzNi2TkY1OakiuoUOjU= +cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= +cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= +cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= +cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY= +cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E= +cloud.google.com/go/vision/v2 v2.6.0/go.mod h1:158Hes0MvOS9Z/bDMSFpjwsUrZ5fPrdwuyyvKSGAGMY= +cloud.google.com/go/vision/v2 v2.7.0/go.mod h1:H89VysHy21avemp6xcf9b9JvZHVehWbET0uT/bcuY/0= +cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE= +cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g= +cloud.google.com/go/vmmigration v1.5.0/go.mod h1:E4YQ8q7/4W9gobHjQg4JJSgXXSgY21nA5r8swQV+Xxc= +cloud.google.com/go/vmmigration v1.6.0/go.mod h1:bopQ/g4z+8qXzichC7GW1w2MjbErL54rk3/C843CjfY= +cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN4sagiox+OI208= +cloud.google.com/go/vmwareengine v0.2.2/go.mod h1:sKdctNJxb3KLZkE/6Oui94iw/xs9PRNC2wnNLXsHvH8= +cloud.google.com/go/vmwareengine v0.3.0/go.mod h1:wvoyMvNWdIzxMYSpH/R7y2h5h3WFkx6d+1TIsP39WGY= +cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= +cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= +cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27RV31lnmZes= +cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= +cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= +cloud.google.com/go/webrisk v1.6.0/go.mod h1:65sW9V9rOosnc9ZY7A7jsy1zoHS5W9IAXv6dGqhMQMc= +cloud.google.com/go/webrisk v1.7.0/go.mod h1:mVMHgEYH0r337nmt1JyLthzMr6YxwN1aAIEc2fTcq7A= +cloud.google.com/go/webrisk v1.8.0/go.mod h1:oJPDuamzHXgUc+b8SiHRcVInZQuybnvEW72PqTc7sSg= +cloud.google.com/go/websecurityscanner v1.3.0/go.mod h1:uImdKm2wyeXQevQJXeh8Uun/Ym1VqworNDlBXQevGMo= +cloud.google.com/go/websecurityscanner v1.4.0/go.mod h1:ebit/Fp0a+FWu5j4JOmJEV8S8CzdTkAS77oDsiSqYWQ= +cloud.google.com/go/websecurityscanner v1.5.0/go.mod h1:Y6xdCPy81yi0SQnDY1xdNTNpfY1oAgXUlcfN3B3eSng= +cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= +cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= +cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= +cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= +cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go v55.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= +git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= -github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= -github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20200415212048-7901bc822317/go.mod h1:DF8FZRxMHMGv/vP2lQP6h+dYzzjpuRn24VeRiYn3qjQ= -github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA= -github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp8u+gxLtPgKGjk5hCxuy2hrRejBTA9xFU= -github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= -github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= -github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= -github.com/Microsoft/hcsshim v0.8.10-0.20200715222032-5eafd1556990/go.mod h1:ay/0dTb7NsG8QMDfsRfLHgZo/6xAJShLe1+ePPflihk= +github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= +github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= +github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= +github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= +github.com/alecthomas/kingpin/v2 v2.3.1/go.mod h1:oYL5vtsvEHZGHxU7DMp32Dvx+qL+ptGn6lWaot2vCNE= +github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/alibaba-cloud-sdk-go v1.62.156 h1:K4N91T1+RlSlx+t2dujeDviy4ehSGVjEltluDgmeHS4= github.com/aliyun/alibaba-cloud-sdk-go v1.62.156/go.mod h1:Api2AkmMgGaSUAhmk76oaFObkoeCPc/bKAqcyplPODs= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= +github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= +github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= +github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/auth0/go-jwt-middleware v1.0.1/go.mod h1:YSeUX3z6+TF2H+7padiEqNJ73Zy9vXW72U//IgN0BIM= -github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= -github.com/aws/aws-sdk-go v1.38.49/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 h1:7aWHqerlJ41y6FOsEUvknqgXnGmJyJSbjhAWq5pO4F8= -github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= -github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= +github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= +github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= -github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/clusterhq/flocker-go v0.0.0-20160920122132-2b8b7259d313/go.mod h1:P1wt9Z3DP8O6W3rvwCt0REIlshg1InHImaLW0t3ObY0= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= -github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/container-storage-interface/spec v1.5.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s= -github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= -github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= -github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= -github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= -github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= -github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= -github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/coredns/caddy v1.1.0/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4= -github.com/coredns/corefile-migration v1.0.12/go.mod h1:NJOI8ceUF/NTgEwtjD+TUq3/BnH/GF7WAM3RzCa3hBo= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.3.1/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= -github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= -github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= -github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= -github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v20.10.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/edgexfoundry/go-mod-core-contracts/v3 v3.0.0 h1:xjwCI34DLM31cSl1q9XmYgXS3JqXufQJMgohnLLLDx0= github.com/edgexfoundry/go-mod-core-contracts/v3 v3.0.0/go.mod h1:zzzWGWij6wAqm1go9TLs++TFMIsBqBb1eRnIj4mRxGw= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.11.2-0.20200112161605-a7c079c43d51+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.16.0+incompatible h1:rgqiKNjTnFQA6kkhFe16D8epTksy9HQ1MyrbDXSdYhM= -github.com/emicklei/go-restful v2.16.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= +github.com/envoyproxy/go-control-plane v0.11.1-0.20230524094728-9239064ad72f/go.mod h1:sfYdkwUW4BA3PbKjySwjJy+O4Pu0h62rlqCMHNk+K+Q= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= +github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= github.com/envoyproxy/protoc-gen-validate v0.10.1 h1:c0g45+xCJhdgFGw7a5QAfdS4byAbud7miNWJ1WwEVf8= -github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= -github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= -github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= -github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= -github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/fvbommel/sortorder v1.0.1 h1:dSnXLt4mJYH25uDDGa3biZNQsozaUWDSWeKJ0qqFfzE= -github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= +github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQmYw= +github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88= github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= -github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= +github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= +github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -228,25 +758,40 @@ github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQr github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= +github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM= -github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= +github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.4-0.20191224164422-1f9748e5f45e/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= -github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= -github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= @@ -258,20 +803,22 @@ github.com/go-resty/resty/v2 v2.12.0 h1:rsVL8P90LFvkUYq/V5BTVe203WfRIU4gvcf+yfzJ github.com/go-resty/resty/v2 v2.12.0/go.mod h1:o0yGPrkS3lOe1+eFajk6kBW8ScXzwU3hD69/gt2yB/0= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -282,8 +829,11 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -301,22 +851,34 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/cadvisor v0.39.2/go.mod h1:kN93gpdevu+bpS227TyHVZyCU5bbqCzTj5T9drl34MI= +github.com/google/cel-go v0.16.1 h1:3hZfSNiAU3KOiNtxuFXVp5WFy4hf/Ly3Sa4/7F8SXNo= +github.com/google/cel-go v0.16.1/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= +github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -324,11 +886,27 @@ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= @@ -336,64 +914,54 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= +github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= +github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= -github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 h1:lLT7ZLSzGLI08vc9cpd+tYmNWjdKDqyr/2L+f6U12Fk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/heketi/heketi v10.3.0+incompatible/go.mod h1:bB9ly3RchcQqsQ9CpyaQwvva7RS5ytVoSoholZQON6o= -github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6/go.mod h1:xGMAM8JLi7UkZt1i4FQeQy0R2T8GLUwQhOP5M1gBhy4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/ishidawataru/sctp v0.0.0-20190723014705-7c296d48a2b5/go.mod h1:DM4VvS+hD/kDi1U1QsX2fnZowwBhqD0Dk3bRPKF/Oc8= github.com/jarcoal/httpmock v1.3.0 h1:2RJ8GP0IIaWwcC9Fp2BmVi8Kog3v2Hn7VXM3fTd+nuc= github.com/jarcoal/httpmock v1.3.0/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -402,9 +970,7 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= -github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -416,77 +982,67 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= -github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kubernetes/kubernetes v1.22.3 h1:0gYnqsr5nZiAO+iDkEU7RJ6Ne2CMyoinJXVm5qVSTiE= -github.com/kubernetes/kubernetes v1.22.3/go.mod h1:Snea7fgIObGgHmLbUJ3OgjGEr5bjj16iEdp5oHS6eS8= +github.com/kubernetes/kubernetes v1.28.9 h1:P8pv52Uy7kaMAQIloaQtqGcwMItIAbFhP1o5qbk7aCk= +github.com/kubernetes/kubernetes v1.28.9/go.mod h1:chlmcCDBnOA/y+572cw8dO0Rci1wiA8bm5+zhPdFLCk= github.com/leodido/go-urn v1.2.3 h1:6BE2vPT0lqoz3fmOesHZiaiFh7889ssCo2GMvLCfiuA= github.com/leodido/go-urn v1.2.3/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= -github.com/libopenstorage/openstorage v1.0.0/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= -github.com/lpabon/godbc v0.1.1/go.mod h1:Jo9QV0cf3U6jZABgiJ2skINAXb9j8m51r07g4KI92ZA= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989/go.mod h1:2eu9pRWp8mo84xCg6KswZ+USQHjwgRhNp06sozOdsTY= -github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= -github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/moby/ipvs v1.0.1/go.mod h1:2pngiyseZbIKXNv7hsKj3O9UEz30c53MT9005gt2hxQ= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= -github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= -github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 h1:yH0SvLzcbZxcJXho2yh7CqdENGMQe73Cw3woZBpPli0= -github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= +github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= +github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -494,162 +1050,145 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mohae/deepcopy v0.0.0-20170603005431-491d3605edfb/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mvdan/xurls v1.1.0/go.mod h1:tQlNn3BED8bE/15hnSL2HLkDeLWpNPAwtw7wkEq44oU= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= +github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= +github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= +github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= +github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= +github.com/onsi/ginkgo/v2 v2.8.1/go.mod h1:N1/NbDngAFcSLdyZ+/aYTYGSlq9qMCS/cNKGJjy+csc= +github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxmrTcxyk= +github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= +github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= +github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM= +github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= -github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= +github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= +github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= +github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= +github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw= +github.com/onsi/gomega v1.27.3/go.mod h1:5vG284IBtfDAmDyrK+eGyZmUgUlmi+Wngqo557cZ6Gw= +github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc95/go.mod h1:z+bZxa/+Tz/FmYVWkhUajJdzFeOqjc5vrqskhVyHGUM= -github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= -github.com/opencontainers/runc v1.1.12 h1:BOIssBaW1La0/qbNZHXOOa71dZfZEQOzW7dqQf3phss= -github.com/opencontainers/runc v1.1.12/go.mod h1:S+lQwSfncpBha7XTy/5lBwWgm5+y5Ma/O44Ekby9FK8= -github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= -github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU= github.com/openyurtio/apiserver-network-proxy v0.1.0 h1:uJI6LeAHmkQL0zV1+NIbgRsx2ayzsPfMA2bd1gROypc= github.com/openyurtio/apiserver-network-proxy v0.1.0/go.mod h1:X5Au3jBNIgYL2uK0IHeNGnZqlUlVSCFQhi/npPgkKRg= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= +github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/projectcalico/api v0.0.0-20230222223746-44aa60c2201f h1:7+GPMHkGC2rlL/Le/hdDKBkIwhtBuKU467KxgLg8V34= github.com/projectcalico/api v0.0.0-20230222223746-44aa60c2201f/go.mod h1:Avoy1rTN1GfeisnHGf3WhQNqR+BuGOcwfNFsdWX6OHE= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/quobyte/api v0.1.8/go.mod h1:jL7lIHrmqQ7yh05OJ+eEEdHr0u/kmT1Ff9iHd+4H6VI= -github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= -github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= -github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= +github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= +github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= -github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= -github.com/storageos/go-api v2.2.0+incompatible/go.mod h1:ZrLn+e0ZuF3Y65PNF6dIwbJPZqfmtCXxFm9ckv0agOY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -661,114 +1200,111 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= -github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= -github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= +github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= +github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xhit/go-str2duration v1.2.0/go.mod h1:3cPSlfZlUHVlneIVfePFWcJZsuwf+P1v2SRTV4cUmp4= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= -github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.etcd.io/etcd/api/v3 v3.5.0 h1:GsV3S+OfZEOCNXdtNkBSR7kgLobAa/SO6tCxRa0GAYw= -go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.0 h1:2aQv6F436YnN7I4VbI8PPYrBhu+SmrTaADcf8Mi/6PU= -go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.0 h1:ftQ0nOOHMcbMS3KIaDQ0g5Qcd6bhaBrQT6b89DfwLTs= -go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.etcd.io/etcd/client/v3 v3.5.0 h1:62Eh0XOro+rDwkrypAGDfgmNh5Joq+z+W9HZdlXMzek= -go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= -go.etcd.io/etcd/pkg/v3 v3.5.0 h1:ntrg6vvKRW26JRmHTE0iNlDgYK6JX3hg/4cD62X0ixk= -go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= -go.etcd.io/etcd/raft/v3 v3.5.0 h1:kw2TmO3yFTgE+F0mdKkG7xMxkit2duBDa2Hu6D/HMlw= -go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= -go.etcd.io/etcd/server/v3 v3.5.0 h1:jk8D/lwGEDlQU9kZXUFMSANkE22Sg5+mW27ip8xcF9E= -go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= +go.etcd.io/etcd/api/v3 v3.5.9 h1:4wSsluwyTbGGmyjJktOf3wFQoTBIURXHnq9n/G/JQHs= +go.etcd.io/etcd/api/v3 v3.5.9/go.mod h1:uyAal843mC8uUVSLWz6eHa/d971iDGnCRpmKd2Z+X8k= +go.etcd.io/etcd/client/pkg/v3 v3.5.9 h1:oidDC4+YEuSIQbsR94rY9gur91UPL6DnxDCIYd2IGsE= +go.etcd.io/etcd/client/pkg/v3 v3.5.9/go.mod h1:y+CzeSmkMpWN2Jyu1npecjB9BBnABxGM4pN8cGuJeL4= +go.etcd.io/etcd/client/v2 v2.305.9 h1:YZ2OLi0OvR0H75AcgSUajjd5uqKDKocQUqROTG11jIo= +go.etcd.io/etcd/client/v3 v3.5.9 h1:r5xghnU7CwbUxD/fbUtRyJGaYNfDun8sp/gTr1hew6E= +go.etcd.io/etcd/client/v3 v3.5.9/go.mod h1:i/Eo5LrZ5IKqpbtpPDuaUnDOUv471oDg8cjQaUr2MbA= +go.etcd.io/etcd/pkg/v3 v3.5.9 h1:6R2jg/aWd/zB9+9JxmijDKStGJAPFsX3e6BeJkMi6eQ= +go.etcd.io/etcd/raft/v3 v3.5.9 h1:ZZ1GIHoUlHsn0QVqiRysAm3/81Xx7+i2d7nSdWxlOiI= +go.etcd.io/etcd/server/v3 v3.5.9 h1:vomEmmxeztLtS5OEH7d0hBAg4cjVIu9wXuNzUZx2ZA0= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0= -go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 h1:sO4WKdPAudZGKPcpZT4MJn6JaDmpyLrMPDGGyA1SttE= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 h1:Q3C9yzW6I9jqEc8sawxzxZmY48fs9u220KXq6d5s3XU= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= -go.opentelemetry.io/otel v0.20.0 h1:eaP0Fqu7SXHwvjiqDq83zImeehOHX8doTvU9AwXON8g= -go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg= -go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= -go.opentelemetry.io/otel/metric v0.20.0 h1:4kzhXFP+btKm4jwxpjIqjs41A7MakRFUS86bqLHTIw8= -go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/oteltest v0.20.0 h1:HiITxCawalo5vQzdHfKeZurV8x7ljcqAgiWzF6Vaeaw= -go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= -go.opentelemetry.io/otel/sdk v0.20.0 h1:JsxtGXd06J8jrnya7fdI/U/MR6yXA5DtbZy+qoHQlr8= -go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk/export/metric v0.20.0 h1:c5VRjxCXdQlx1HjzwGdQHzZaVI82b5EbBgOu2ljD92g= -go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= -go.opentelemetry.io/otel/sdk/metric v0.20.0 h1:7ao1wpzHRVKf0OQ7GIxiQJA6X7DLX9o14gmVon7mMK8= -go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= -go.opentelemetry.io/otel/trace v0.20.0 h1:1DL6EXUdcg95gukhuRRvLDO/4X5THh/5dIV52lqtnbw= -go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= -go.opentelemetry.io/proto/otlp v0.7.0 h1:rwOQPCuKAKmwGKq2aVNnYIibI6wnV7EvzgfTCzcdGg8= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 h1:xFSRQBbXF6VvYRf2lqMJXxoB72XI1K/azav8TekHHSw= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0/go.mod h1:h8TWwRAhQpOd0aM5nYsRD8+flnkj+526GEIVlarH7eY= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.1 h1:sxoY9kG1s1WpSYNyzm24rlwH4lnRYFXUVVBmKMBfRgw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.1/go.mod h1:9NiG9I2aHTKkcxqCILhjtyNA1QEiCjdBACv4IvrFQ+c= +go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRMp9yvM= +go.opentelemetry.io/otel v1.10.0 h1:Y7DTJMR6zs1xkS/upamJYk0SxxN4C9AqRd77jmZnyY4= +go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 h1:TaB+1rQhddO1sF71MpZOZAuSPW1klK2M8XxfrBMfK7Y= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0/go.mod h1:78XhIg8Ht9vR4tbLNUhXsiOnE2HOuSeKAiAcoVQEpOY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 h1:pDDYmo0QadUPal5fwXoY1pmMpFcdyhXOmL5drCrI3vU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0/go.mod h1:Krqnjl22jUJ0HgMzw5eveuCvFDXY4nSYb4F8t5gdrag= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 h1:KtiUEhQmj/Pa874bVYKGNVdq8NPKiacPbaRRtgXi+t4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0/go.mod h1:OfUCyyIiDvNXHWpcWgbF+MWvqPZiNa3YDEnivcnYsV0= +go.opentelemetry.io/otel/metric v0.31.0 h1:6SiklT+gfWAwWUR0meEMxQBtihpiEs4c+vL9spDTqUs= +go.opentelemetry.io/otel/metric v0.31.0/go.mod h1:ohmwj9KTSIeBnDBm/ZwH2PSZxZzoOaG2xZeekTRzL5A= +go.opentelemetry.io/otel/sdk v1.10.0 h1:jZ6K7sVn04kk/3DNUdJ4mqRlGDiXAVuIG+MMENpTNdY= +go.opentelemetry.io/otel/sdk v1.10.0/go.mod h1:vO06iKzD5baltJz1zarxMCNHFpUlUiOy4s65ECtn6kE= +go.opentelemetry.io/otel/trace v1.8.0/go.mod h1:0Bt3PXY8w+3pheS3hQUt+wow8b1ojPaTBoTCh2zIFI4= +go.opentelemetry.io/otel/trace v1.10.0 h1:npQMbR8o7mum8uF95yFbOEJffhs1sbCOfDh8zAJiH5E= +go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= -go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY= +go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= +go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= @@ -777,20 +1313,30 @@ golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20210220032938-85be41e4509f/go.mod h1:I6l2HNBLBZEcrOoCpyKLdY2lHoRZ8lI4x60KMCQDft4= +golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 h1:tnebWN09GYg9OLPss1KXj8txwZc6X6uMr6VFdcGNbHw= +golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -801,38 +1347,44 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mobile v0.0.0-20201217150744-e6ae53a27f4f/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -843,51 +1395,112 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= +golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -895,24 +1508,17 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -920,41 +1526,97 @@ golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= @@ -967,41 +1629,43 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1009,7 +1673,6 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1017,27 +1680,64 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= -gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= -gonum.org/v1/gonum v0.6.2/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= +gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= +gonum.org/v1/plot v0.10.1/go.mod h1:VZW5OlhkL1mysU9vaqNHnsy86inf6Ot+jB3r+BczCEo= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1045,15 +1745,62 @@ google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEn google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.1-0.20200106000736-b8fc810ca6b5/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= +google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= +google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= +google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= +google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI= +google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08= +google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= +google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= +google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= +google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -1069,41 +1816,178 @@ google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= +google.golang.org/genproto v0.0.0-20220801145646-83ce21fca29f/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= +google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220817144833-d7fd3f11b9b1/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829144015-23454907ede3/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220913154956-18f8339a66a5/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220914142337-ca0e39ece12f/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006/go.mod h1:ht8XFiar2npT/g4vkk7O0WYS1sHOHbdujxbEp7CJWbw= +google.golang.org/genproto v0.0.0-20220926165614-551eb538f295/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= +google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= +google.golang.org/genproto v0.0.0-20221109142239-94d6d90a7d66/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221114212237-e4508ebdbee1/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221201204527-e3fa12d562f3/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230112194545-e10362b5ecf9/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230113154510-dbe35b8444a5/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230123190316-2c411cf9d197/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230124163310-31e0e69b6fc2/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230125152338-dcaf20b6aeaa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230127162408-596548ed4efa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA= +google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY= google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 h1:9NWlQfY2ePejTmfwUH1OWwmznFa+0kKcHGPDvcPza9M= google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 h1:m8v1xLLLzMe1m5P+gCTF8nJB9epwZQUBERm20Oy1poQ= google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= +google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc v1.57.1 h1:upNTNqv0ES+2ZOOqACwVtS3Il8M12/+Hz41RCPzAjQg= google.golang.org/grpc v1.57.1/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1117,34 +2001,32 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.1/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1159,92 +2041,104 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.22.3 h1:wOoES2GoSkUsdped2RB4zYypPqWtvprGoKCENTOOjP4= -k8s.io/api v0.22.3/go.mod h1:azgiXFiXqiWyLCfI62/eYBOu19rj2LKmIhFPP4+33fs= -k8s.io/apiextensions-apiserver v0.22.3 h1:bKku7MqawIbtTZc084BZoMV4fz0WZuvCnB5E+yrQXGM= -k8s.io/apiextensions-apiserver v0.22.3/go.mod h1:f4plF+CXeqI89jAXL0Ml4LI/kSAZ54JS94+XOX1sae8= -k8s.io/apimachinery v0.22.3 h1:mrvBG5CZnEfwgpVqWcrRKvdsYECTrhAR6cApAgdsflk= -k8s.io/apimachinery v0.22.3/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/apiserver v0.22.3 h1:x21xyLQ2qvPr5vjOTVOBaSJu8svnU2wfLOfSjNJEOdw= -k8s.io/apiserver v0.22.3/go.mod h1:oam7lH/F1Kto/WTamyQYrD68fS0mGUBORAFf6x/9Mxs= -k8s.io/cli-runtime v0.22.3 h1:AeOgaDpb/k36amWsjyyIU+FLpLzzdmoLD5gn38c5fio= -k8s.io/cli-runtime v0.22.3/go.mod h1:um6JvCxV9Hrhq0zCUxcqYoY7/wF64g6IYgOViI8sg6Q= -k8s.io/client-go v0.22.3 h1:6onkOSc+YNdwq5zXE0wFXicq64rrym+mXwHu/CPVGO4= -k8s.io/client-go v0.22.3/go.mod h1:ElDjYf8gvZsKDYexmsmnMQ0DYO8W9RwBjfQ1PI53yow= -k8s.io/cloud-provider v0.22.3 h1:ZsWdB0WmyjKlE901EM14BuSvnN+QPGrCGjcfDc+b5NI= -k8s.io/cloud-provider v0.22.3/go.mod h1:GsKMR5EnNH4zcfkEvOxBPEZVuRvadVRkZvGqYxxBvO4= -k8s.io/cluster-bootstrap v0.22.3 h1:uTrzquwoXsstQ6PCea0dYbKWcPCetMp4MZEkZbT+Ei0= -k8s.io/cluster-bootstrap v0.22.3/go.mod h1:FVBAeGJ/T6QbNgGb7DX98FCjExJnNLsRXtGRMjEQ26I= -k8s.io/code-generator v0.22.3/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= -k8s.io/component-base v0.22.3 h1:/+hryAW03u3FpJQww+GSMsArJNUbGjH66lrgxaRynLU= -k8s.io/component-base v0.22.3/go.mod h1:kuybv1miLCMoOk3ebrqF93GbQHQx6W2287FC0YEQY6s= -k8s.io/component-helpers v0.22.3 h1:08tn+T8HnjRTwDP2ErIBhHGvPcYJf5zWaWW83golHWc= -k8s.io/component-helpers v0.22.3/go.mod h1:7OVySVH5elhHKuJKUOxZEfpT1Bm3ChmBQZHmuFfbGHk= -k8s.io/controller-manager v0.22.3 h1:nBKG8MsgtUd/oFaZvE5zAYRIr45+Hn8QkHzq5+CtPOE= -k8s.io/controller-manager v0.22.3/go.mod h1:4cvQGMvYf6IpTY08/NigEiI5UrN/cbtOe5e5WepYmcQ= -k8s.io/cri-api v0.22.3/go.mod h1:mj5DGUtElRyErU5AZ8EM0ahxbElYsaLAMTPhLPQ40Eg= -k8s.io/csi-translation-lib v0.22.3/go.mod h1:YkdI+scWhZJQeA26iNg9XrKO3LhLz6dAcRKsL0RIiUY= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +k8s.io/api v0.28.9 h1:E7VEXXCAlSrp+08zq4zgd+ko6Ttu0Mw+XoXlIkDTVW0= +k8s.io/api v0.28.9/go.mod h1:AnCsDYf3SHjfa8mPG5LGYf+iF4mie+3peLQR51MMCgw= +k8s.io/apiextensions-apiserver v0.28.9 h1:yzPHp+4IASHeu7XIPkAKJrY4UjWdjiAjOcQMd6oNKj0= +k8s.io/apiextensions-apiserver v0.28.9/go.mod h1:Rjhvq5y3JESdZgV2UOByldyefCfRrUguVpBLYOAIbVs= +k8s.io/apimachinery v0.28.9 h1:aXz4Zxsw+Pk4KhBerAtKRxNN1uSMWKfciL/iOdBfXvA= +k8s.io/apimachinery v0.28.9/go.mod h1:zUG757HaKs6Dc3iGtKjzIpBfqTM4yiRsEe3/E7NX15o= +k8s.io/apiserver v0.28.9 h1:koPXvgSXRBDxKJQjJGdZNgPsT9lQv6scJJFipd1m86E= +k8s.io/apiserver v0.28.9/go.mod h1:D51I37WBZojJhmLcjNVE4GSVrjiUHP+yq+N5KvKn2wY= +k8s.io/cli-runtime v0.28.9 h1:TfEV/UgCiXewliUHOHsUMZ1bfENhqcqKkA/hqQ/HwvQ= +k8s.io/cli-runtime v0.28.9/go.mod h1:PgxW97xCDbtWgsuo2nahMc2/MxcSDgscdwm8XZ7973A= +k8s.io/client-go v0.28.9 h1:mmMvejwc/KDjMLmDpyaxkWNzlWRCJ6ht7Qsbsnwn39Y= +k8s.io/client-go v0.28.9/go.mod h1:GFDy3rUNId++WGrr0hRaBrs+y1eZz5JtVZODEalhRMo= +k8s.io/cloud-provider v0.28.9 h1:FBW4Ii1NdXCHKprzkM8/s5BpxvLgJmYrZTNJABsVX7Y= +k8s.io/cloud-provider v0.28.9/go.mod h1:7tFyiftAlSARvJS6mzZQQKKDQA81asNQ2usg35R3Exo= +k8s.io/cluster-bootstrap v0.28.9 h1:MxyJszYYyWEGNrmkv/vxZ8HJUgmb1ACS9PzMb7xzrn4= +k8s.io/cluster-bootstrap v0.28.9/go.mod h1:feeH01O2+GaGfi86gzQh0JpevSyzuXOg0TXj/UHGLdE= +k8s.io/component-base v0.28.9 h1:ySM2PR8Z/xaUSG1Akd3yM6dqUezTltI7S5aV41MMuuc= +k8s.io/component-base v0.28.9/go.mod h1:QtWzscEhCKRfHV24/S+11BwWjVxhC6fd3RYoEgZcWFU= +k8s.io/component-helpers v0.28.9 h1:knX9F2nRoxF4wplgXO4C5tE4/k7HGszK3177Tm4+CUc= +k8s.io/component-helpers v0.28.9/go.mod h1:TdAkLbywEDE2CB5h8LbM/W03T3k8wvqAaoPcEZrr6Z4= +k8s.io/controller-manager v0.28.9 h1:muAtmO2mDN7pDkAJQMknvWy+WQhkvvi/jK1V82+qbLw= +k8s.io/controller-manager v0.28.9/go.mod h1:RYP65K6GWLRWYZR7PRRaStfvgeXkhCGZwJsxRPuaDV0= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= -k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/kube-aggregator v0.22.3/go.mod h1:TIpLq1HvR/S4y75i3y+4q9ik3ZvgyaDz72CBfDS0A6E= -k8s.io/kube-controller-manager v0.22.3 h1:DatYcgMKAn28e2A7MiMULoRoft3SaCV/qVk+FoGTUw0= -k8s.io/kube-controller-manager v0.22.3/go.mod h1:7biFk6Azf7xD+pzTScw7X9M5vGScqYp4J4wOT61QL1s= -k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4= +k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= +k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kms v0.28.9 h1:ApCWJulBl+uFRTr2jtTpG1lffmqqMuLnOH/RUbtO4UY= +k8s.io/kms v0.28.9/go.mod h1:VgyAIRMFqZX9lHyixecU/JTI0wnPD1wCIlquvlXRJ+Y= +k8s.io/kube-controller-manager v0.28.9 h1:VDe2umkomaj9KQYREEsEia2yLcBLQ8bmiAClCt2GJog= +k8s.io/kube-controller-manager v0.28.9/go.mod h1:C0ZKOOuWPtpTRrCpq+SjPXlp8JR79Zp7bN5Unk9Vm/I= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kube-proxy v0.22.3/go.mod h1:9ta1U8GKKo6by981sN/L6MhFJzPWxMdfh7plVPH1I2s= -k8s.io/kube-scheduler v0.22.3/go.mod h1:jVLHSttd8cSejBLOeiWE+g8etA6XdOBGiR8tI577OhU= -k8s.io/kubectl v0.22.3 h1:xziSHHyFHg2nt9vE6A0XqW5dOePNSlzxG8z3z+IY63E= -k8s.io/kubectl v0.22.3/go.mod h1:gcpQHPOx+Jke9Og6Li7YxR/ZuaOtFUeJw7xHH617tHs= -k8s.io/kubelet v0.22.3 h1:C21Kg66Zzvc21uJITEPg4stGMcSZsR1JB+7+6Uwm8zs= -k8s.io/kubelet v0.22.3/go.mod h1:9nUZNGUigU2uAIm7kgf8BsvYDI9KjIE5nt9+yI1+p7w= -k8s.io/legacy-cloud-providers v0.22.3/go.mod h1:eEOOaRtP2PuCVkjZvuTPa6ZgyPpzJkCVqpE3YtuArLQ= -k8s.io/metrics v0.22.3/go.mod h1:HbLFLRKtXzoC/6tHLQAlO9AeOBXZp2eB6SsgkbujoNI= -k8s.io/mount-utils v0.22.3/go.mod h1:dHl6c2P60T5LHUnZxVslyly9EDCMzvhtISO5aY+Z4sk= -k8s.io/pod-security-admission v0.22.3/go.mod h1:xtkf/UhVWICokQLSDvD+8plfGkTQW4VTJvnixVWCeWk= -k8s.io/sample-apiserver v0.22.3/go.mod h1:HuEOdD/pT5R7gKNr2REb62uabZaJuFZyY3wUd86nFCA= -k8s.io/system-validators v1.5.0/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= +k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= +k8s.io/kubectl v0.28.9 h1:FTf/aapuuFxPmt8gYUeqUmcsgG0gKC2ei6n+TO5sGOw= +k8s.io/kubectl v0.28.9/go.mod h1:ip/zTUr1MM/H2M+YbPHnSKLt0x6kb85SJtRSjwEGDfs= +k8s.io/kubelet v0.28.9 h1:76v00fFLeniz27kXhGGUIxONdwa9LKcD2Jd5cXYAZko= +k8s.io/kubelet v0.28.9/go.mod h1:46P39DFjI+E59nU2OgpatyS3oWy58ClulKO6riZ/97o= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b h1:wxEMGetGMur3J1xuGLQY7GEQYg9bZxKn3tKo5k/eYcs= -k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= -modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= -modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= -modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= -modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= +modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= +modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws= +modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo= +modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= +modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= +modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A= +modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU= +modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= +modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0= +modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= +modernc.org/memory v1.2.0/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= +modernc.org/memory v1.2.1/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4= +modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= +modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= +modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw= +modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22 h1:fmRfl9WJ4ApJn7LxNuED4m0t18qivVQOxP6aAYG9J6c= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-runtime v0.10.3 h1:s5Ttmw/B4AuIbwrXD3sfBkXwnPMMWrqpVj4WRt1dano= -sigs.k8s.io/controller-runtime v0.10.3/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= -sigs.k8s.io/kustomize/api v0.8.11 h1:LzQzlq6Z023b+mBtc6v72N2mSHYmN8x7ssgbf/hv0H8= -sigs.k8s.io/kustomize/api v0.8.11/go.mod h1:a77Ls36JdfCWojpUqR6m60pdGY1AYFix4AH83nJtY1g= -sigs.k8s.io/kustomize/cmd/config v0.9.13/go.mod h1:7547FLF8W/lTaDf0BDqFTbZxM9zqwEJqCKN9sSR0xSs= -sigs.k8s.io/kustomize/kustomize/v4 v4.2.0/go.mod h1:MOkR6fmhwG7hEDRXBYELTi5GSFcLwfqwzTRHW3kv5go= -sigs.k8s.io/kustomize/kyaml v0.11.0 h1:9KhiCPKaVyuPcgOLJXkvytOvjMJLoxpjodiycb4gHsA= -sigs.k8s.io/kustomize/kyaml v0.11.0/go.mod h1:GNMwjim4Ypgp/MueD3zXHLRJEjz7RvtPae0AwlvEMFM= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.2 h1:N2wvoG4CkNqORML7GHY9xkGKxswDhpAD46poBd/hHHg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +sigs.k8s.io/controller-runtime v0.16.5 h1:yr1cEJbX08xsTW6XEIzT13KHHmIyX8Umvme2cULvFZw= +sigs.k8s.io/controller-runtime v0.16.5/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0= +sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3/go.mod h1:9n16EZKMhXBNSiUC5kSdFQJkdH3zbxS/JoO619G1VAY= +sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 h1:W6cLQc5pnqM7vh3b7HvGNfXrJ/xL6BDMS0v1V/HHg5U= +sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3/go.mod h1:JWP1Fj0VWGHyw3YUPjXSQnRnrwezrZSrApfX5S0nIag= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/hack/dockerfiles/release/Dockerfile.node-servant b/hack/dockerfiles/release/Dockerfile.node-servant index 53b5b8acf46..9717f2f8380 100644 --- a/hack/dockerfiles/release/Dockerfile.node-servant +++ b/hack/dockerfiles/release/Dockerfile.node-servant @@ -1,6 +1,6 @@ # multi-arch image building for yurt-node-servant -FROM --platform=${BUILDPLATFORM} golang:1.18 as builder +FROM --platform=${BUILDPLATFORM} golang:1.20 as builder ADD . /build ARG TARGETOS TARGETARCH GIT_VERSION GOPROXY MIRROR_REPO WORKDIR /build/ diff --git a/hack/dockerfiles/release/Dockerfile.yurt-iot-dock b/hack/dockerfiles/release/Dockerfile.yurt-iot-dock index d73a35cb790..8c51ad908bf 100644 --- a/hack/dockerfiles/release/Dockerfile.yurt-iot-dock +++ b/hack/dockerfiles/release/Dockerfile.yurt-iot-dock @@ -1,6 +1,6 @@ # multi-arch image building for yurt-iot-dock -FROM --platform=${BUILDPLATFORM} golang:1.18 as builder +FROM --platform=${BUILDPLATFORM} golang:1.20 as builder ADD . /build ARG TARGETOS TARGETARCH GIT_VERSION GOPROXY MIRROR_REPO WORKDIR /build/ diff --git a/hack/dockerfiles/release/Dockerfile.yurt-manager b/hack/dockerfiles/release/Dockerfile.yurt-manager index 82961429dc8..9148d12fd62 100644 --- a/hack/dockerfiles/release/Dockerfile.yurt-manager +++ b/hack/dockerfiles/release/Dockerfile.yurt-manager @@ -1,6 +1,6 @@ # multi-arch image building for yurt-tunnel-server -FROM --platform=${BUILDPLATFORM} golang:1.18 as builder +FROM --platform=${BUILDPLATFORM} golang:1.20 as builder ADD . /build ARG TARGETOS TARGETARCH GIT_VERSION GOPROXY MIRROR_REPO WORKDIR /build/ diff --git a/hack/dockerfiles/release/Dockerfile.yurt-tunnel-agent b/hack/dockerfiles/release/Dockerfile.yurt-tunnel-agent index 112d192cb8f..ac83d396e7c 100644 --- a/hack/dockerfiles/release/Dockerfile.yurt-tunnel-agent +++ b/hack/dockerfiles/release/Dockerfile.yurt-tunnel-agent @@ -1,6 +1,6 @@ # multi-arch image building for yurt-tunnel-agent -FROM --platform=${BUILDPLATFORM} golang:1.18 as builder +FROM --platform=${BUILDPLATFORM} golang:1.20 as builder ADD . /build ARG TARGETOS TARGETARCH GIT_VERSION GOPROXY MIRROR_REPO WORKDIR /build/ diff --git a/hack/dockerfiles/release/Dockerfile.yurt-tunnel-server b/hack/dockerfiles/release/Dockerfile.yurt-tunnel-server index 860951e7b79..436aafc303a 100644 --- a/hack/dockerfiles/release/Dockerfile.yurt-tunnel-server +++ b/hack/dockerfiles/release/Dockerfile.yurt-tunnel-server @@ -1,6 +1,6 @@ # multi-arch image building for yurt-tunnel-server -FROM --platform=${BUILDPLATFORM} golang:1.18 as builder +FROM --platform=${BUILDPLATFORM} golang:1.20 as builder ADD . /build ARG TARGETOS TARGETARCH GIT_VERSION GOPROXY MIRROR_REPO WORKDIR /build/ diff --git a/hack/dockerfiles/release/Dockerfile.yurthub b/hack/dockerfiles/release/Dockerfile.yurthub index 53b83d1e117..d6738143e0e 100644 --- a/hack/dockerfiles/release/Dockerfile.yurthub +++ b/hack/dockerfiles/release/Dockerfile.yurthub @@ -1,6 +1,6 @@ # multi-arch image building for yurthub -FROM --platform=${BUILDPLATFORM} golang:1.18 as builder +FROM --platform=${BUILDPLATFORM} golang:1.20 as builder ADD . /build ARG TARGETOS TARGETARCH GIT_VERSION GOPROXY MIRROR_REPO WORKDIR /build/ diff --git a/hack/make-rules/image_build.sh b/hack/make-rules/image_build.sh index 1f4afa71d07..8a7e9540eaa 100755 --- a/hack/make-rules/image_build.sh +++ b/hack/make-rules/image_build.sh @@ -36,7 +36,7 @@ IMAGE_REPO=${IMAGE_REPO:-"openyurt"} IMAGE_TAG=${IMAGE_TAG:-$(get_image_tag)} DOCKER_BUILD_ARGS="" DOCKER_EXTRA_ENVS="" -BUILD_BASE_IMAGE="golang:1.18" +BUILD_BASE_IMAGE="golang:1.20" BUILD_GOPROXY=$(go env GOPROXY) GOPROXY_CN="https://goproxy.cn" APKREPO_MIRROR_CN="mirrors.aliyun.com" @@ -79,9 +79,8 @@ docker run \ --env GOARCH=${TARGETARCH} \ --env GOCACHE=/tmp/ \ ${DOCKER_EXTRA_ENVS} \ - --user $(id -u ${USER}):$(id -g ${USER}) \ ${BUILD_BASE_IMAGE} \ - ./hack/make-rules/build.sh ${targets[@]} + /bin/bash -c "git config --global --add safe.directory /build && ./hack/make-rules/build.sh ${targets[@]}" # build images for image in ${targets[@]}; do diff --git a/hack/make-rules/local-up-openyurt.sh b/hack/make-rules/local-up-openyurt.sh index 88c709b0b4f..dfc7e98b081 100755 --- a/hack/make-rules/local-up-openyurt.sh +++ b/hack/make-rules/local-up-openyurt.sh @@ -62,7 +62,7 @@ readonly REQUIRED_IMAGES=( readonly LOCAL_ARCH=$(go env GOHOSTARCH) readonly LOCAL_OS=$(go env GOHOSTOS) readonly CLUSTER_NAME="openyurt-e2e-test" -readonly KUBERNETESVERSION=${KUBERNETESVERSION:-"v1.22"} +readonly KUBERNETESVERSION=${KUBERNETESVERSION:-"v1.28"} readonly NODES_NUM=${NODES_NUM:-3} readonly KIND_KUBECONFIG=${KIND_KUBECONFIG:-${HOME}/.kube/config} readonly DISABLE_DEFAULT_CNI=${DISABLE_DEFAULT_CNI:-"false"} @@ -73,7 +73,7 @@ fi function install_kind { echo "Begin to install kind" - GO111MODULE="on" go get sigs.k8s.io/kind@v0.12.0 + GO111MODULE="on" go install sigs.k8s.io/kind@v0.22.0 } function install_docker { diff --git a/hack/make-rules/run-e2e-tests.sh b/hack/make-rules/run-e2e-tests.sh index 2165082e1d4..defac21fcac 100755 --- a/hack/make-rules/run-e2e-tests.sh +++ b/hack/make-rules/run-e2e-tests.sh @@ -31,6 +31,8 @@ edgeNodeContainer2Name="openyurt-e2e-test-worker2" KUBECONFIG=${KUBECONFIG:-${HOME}/.kube/config} TARGET_PLATFORM=${TARGET_PLATFORMS:-linux/amd64} ENABLE_AUTONOMY_TESTS=${ENABLE_AUTONOMY_TESTS:-true} +USE_LOCAL_CNI=${USE_LOCAL_CNI:-false} +SKIP_SETUP_NETWORK=${SKIP_SETUP_NETWORK:-false} function set_flags() { goldflags="${GOLDFLAGS:--s -w $(project_info)}" @@ -46,12 +48,21 @@ function set_flags() { # set up network function set_up_network() { # set up bridge cni plugins for every node - if [ "$TARGET_PLATFORM" = "linux/amd64" ]; then - wget -O /tmp/cni.tgz https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz + if [ "$USE_LOCAL_CNI" = "true" ]; then + if [ "$TARGET_PLATFORM" = "linux/amd64" ]; then + cp ${YURT_ROOT}/hack/cni/cni-plugins-linux-amd64-v1.4.1.tgz /tmp/cni.tgz + else + cp ${YURT_ROOT}/hack/cni/cni-plugins-linux-arm64-v1.4.1.tgz /tmp/cni.tgz + fi else - wget -O /tmp/cni.tgz https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-arm64-v1.1.1.tgz + if [ "$TARGET_PLATFORM" = "linux/amd64" ]; then + wget -O /tmp/cni.tgz https://github.com/containernetworking/plugins/releases/download/v1.4.1/cni-plugins-linux-amd64-v1.4.1.tgz + else + wget -O /tmp/cni.tgz https://github.com/containernetworking/plugins/releases/download/v1.4.1/cni-plugins-linux-arm64-v1.4.1.tgz + fi fi + docker cp /tmp/cni.tgz $cloudNodeContainerName:/opt/cni/bin/ docker exec -t $cloudNodeContainerName /bin/bash -c 'cd /opt/cni/bin && tar -zxf cni.tgz' @@ -139,23 +150,25 @@ function prepare_autonomy_tests { kubectl wait --for=condition=Ready pod/yurt-e2e-test-nginx-openyurt-e2e-test-worker2 --timeout=${POD_CREATE_TIMEOUT} # set up dig in edge node1 - docker exec -t $edgeNodeContainerName /bin/bash -c "sed -i -r 's/([a-z]{2}.)?archive.ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list" - docker exec -t $edgeNodeContainerName /bin/bash -c "sed -i -r 's/security.ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list" - docker exec -t $edgeNodeContainerName /bin/bash -c "sed -i -r 's/ports.ubuntu.com\/ubuntu-ports/old-releases.ubuntu.com\/ubuntu/g' /etc/apt/sources.list" - docker exec -t $edgeNodeContainerName /bin/bash -c "sed -i -r 's/old-releases.ubuntu.com\/ubuntu-ports/old-releases.ubuntu.com\/ubuntu/g' /etc/apt/sources.list" +# docker exec -t $edgeNodeContainerName /bin/bash -c "sed -i -r 's/([a-z]{2}.)?archive.ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list" +# docker exec -t $edgeNodeContainerName /bin/bash -c "sed -i -r 's/security.ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list" +# docker exec -t $edgeNodeContainerName /bin/bash -c "sed -i -r 's/ports.ubuntu.com\/ubuntu-ports/old-releases.ubuntu.com\/ubuntu/g' /etc/apt/sources.list" +# docker exec -t $edgeNodeContainerName /bin/bash -c "sed -i -r 's/old-releases.ubuntu.com\/ubuntu-ports/old-releases.ubuntu.com\/ubuntu/g' /etc/apt/sources.list" docker exec -t $edgeNodeContainerName /bin/bash -c "apt-get update && apt-get install dnsutils -y" # set up dig in edge node2 - docker exec -t $edgeNodeContainer2Name /bin/bash -c "sed -i -r 's/([a-z]{2}.)?archive.ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list" - docker exec -t $edgeNodeContainer2Name /bin/bash -c "sed -i -r 's/security.ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list" - docker exec -t $edgeNodeContainer2Name /bin/bash -c "sed -i -r 's/ports.ubuntu.com\/ubuntu-ports/old-releases.ubuntu.com\/ubuntu/g' /etc/apt/sources.list" - docker exec -t $edgeNodeContainer2Name /bin/bash -c "sed -i -r 's/old-releases.ubuntu.com\/ubuntu-ports/old-releases.ubuntu.com\/ubuntu/g' /etc/apt/sources.list" +# docker exec -t $edgeNodeContainer2Name /bin/bash -c "sed -i -r 's/([a-z]{2}.)?archive.ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list" +# docker exec -t $edgeNodeContainer2Name /bin/bash -c "sed -i -r 's/security.ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list" +# docker exec -t $edgeNodeContainer2Name /bin/bash -c "sed -i -r 's/ports.ubuntu.com\/ubuntu-ports/old-releases.ubuntu.com\/ubuntu/g' /etc/apt/sources.list" +# docker exec -t $edgeNodeContainer2Name /bin/bash -c "sed -i -r 's/old-releases.ubuntu.com\/ubuntu-ports/old-releases.ubuntu.com\/ubuntu/g' /etc/apt/sources.list" docker exec -t $edgeNodeContainer2Name /bin/bash -c "apt-get update && apt-get install dnsutils -y" } GOOS=${LOCAL_OS} GOARCH=${LOCAL_ARCH} set_flags -set_up_network +if [ "$SKIP_SETUP_NETWORK" != "true" ]; then + set_up_network +fi cleanup diff --git a/pkg/apis/apps/v1alpha1/default.go b/pkg/apis/apps/v1alpha1/default.go index 6d14b6acfac..23c4d6682ee 100644 --- a/pkg/apis/apps/v1alpha1/default.go +++ b/pkg/apis/apps/v1alpha1/default.go @@ -38,7 +38,7 @@ func SetDefaultsNodePool(obj *NodePool) { func SetDefaultsYurtAppSet(obj *YurtAppSet) { if obj.Spec.RevisionHistoryLimit == nil { - obj.Spec.RevisionHistoryLimit = utilpointer.Int32Ptr(10) + obj.Spec.RevisionHistoryLimit = utilpointer.Int32(10) } if obj.Spec.WorkloadTemplate.StatefulSetTemplate != nil { @@ -129,14 +129,14 @@ func SetDefaultPodSpec(in *corev1.PodSpec) { v1.SetDefaults_ResourceList(&a.Resources.Requests) if a.LivenessProbe != nil { v1.SetDefaults_Probe(a.LivenessProbe) - if a.LivenessProbe.Handler.HTTPGet != nil { - v1.SetDefaults_HTTPGetAction(a.LivenessProbe.Handler.HTTPGet) + if a.LivenessProbe.ProbeHandler.HTTPGet != nil { + v1.SetDefaults_HTTPGetAction(a.LivenessProbe.ProbeHandler.HTTPGet) } } if a.ReadinessProbe != nil { v1.SetDefaults_Probe(a.ReadinessProbe) - if a.ReadinessProbe.Handler.HTTPGet != nil { - v1.SetDefaults_HTTPGetAction(a.ReadinessProbe.Handler.HTTPGet) + if a.ReadinessProbe.ProbeHandler.HTTPGet != nil { + v1.SetDefaults_HTTPGetAction(a.ReadinessProbe.ProbeHandler.HTTPGet) } } if a.Lifecycle != nil { @@ -175,14 +175,14 @@ func SetDefaultPodSpec(in *corev1.PodSpec) { v1.SetDefaults_ResourceList(&a.Resources.Requests) if a.LivenessProbe != nil { v1.SetDefaults_Probe(a.LivenessProbe) - if a.LivenessProbe.Handler.HTTPGet != nil { - v1.SetDefaults_HTTPGetAction(a.LivenessProbe.Handler.HTTPGet) + if a.LivenessProbe.ProbeHandler.HTTPGet != nil { + v1.SetDefaults_HTTPGetAction(a.LivenessProbe.ProbeHandler.HTTPGet) } } if a.ReadinessProbe != nil { v1.SetDefaults_Probe(a.ReadinessProbe) - if a.ReadinessProbe.Handler.HTTPGet != nil { - v1.SetDefaults_HTTPGetAction(a.ReadinessProbe.Handler.HTTPGet) + if a.ReadinessProbe.ProbeHandler.HTTPGet != nil { + v1.SetDefaults_HTTPGetAction(a.ReadinessProbe.ProbeHandler.HTTPGet) } } if a.Lifecycle != nil { @@ -239,7 +239,7 @@ func SetDefaultsYurtStaticSet(obj *YurtStaticSet) { func SetDefaultsYurtAppDaemon(obj *YurtAppDaemon) { if obj.Spec.RevisionHistoryLimit == nil { - obj.Spec.RevisionHistoryLimit = utilpointer.Int32Ptr(10) + obj.Spec.RevisionHistoryLimit = utilpointer.Int32(10) } if obj.Spec.WorkloadTemplate.StatefulSetTemplate != nil { diff --git a/pkg/node-servant/components/yurthub.go b/pkg/node-servant/components/yurthub.go index 75cfa04e48c..a4baa538ee5 100644 --- a/pkg/node-servant/components/yurthub.go +++ b/pkg/node-servant/components/yurthub.go @@ -17,6 +17,7 @@ limitations under the License. package components import ( + "context" "fmt" "io" "net/http" @@ -180,7 +181,7 @@ func waitUntilYurthubExit(timeout time.Duration, period time.Duration) error { serverHealthzURL, _ := url.Parse(fmt.Sprintf("http://%s", constants.ServerHealthzServer)) serverHealthzURL.Path = constants.ServerHealthzURLPath - return wait.PollImmediate(period, timeout, func() (bool, error) { + return wait.PollUntilContextTimeout(context.Background(), period, timeout, true, func(ctx context.Context) (bool, error) { _, err := pingClusterHealthz(http.DefaultClient, serverHealthzURL.String()) if err != nil { // means yurthub has exited klog.Infof("yurt-hub is not running, with ping result: %v", err) @@ -200,7 +201,7 @@ func hubHealthcheck(timeout time.Duration) error { serverHealthzURL.Path = constants.ServerHealthzURLPath start := time.Now() - return wait.PollImmediate(hubHealthzCheckFrequency, timeout, func() (bool, error) { + return wait.PollUntilContextTimeout(context.Background(), hubHealthzCheckFrequency, timeout, true, func(ctx context.Context) (bool, error) { _, err := pingClusterHealthz(http.DefaultClient, serverHealthzURL.String()) if err != nil { klog.Infof("yurt-hub is not ready, ping cluster healthz with result: %v", err) diff --git a/pkg/node-servant/static-pod-upgrade/util/pods.go b/pkg/node-servant/static-pod-upgrade/util/pods.go index a13af366232..fcf1983a108 100644 --- a/pkg/node-servant/static-pod-upgrade/util/pods.go +++ b/pkg/node-servant/static-pod-upgrade/util/pods.go @@ -18,7 +18,7 @@ package util import ( "fmt" - "io/ioutil" + "io" "math/rand" "net/http" "time" @@ -65,7 +65,6 @@ func GetPodsFromYurtHub(url string) (*v1.PodList, error) { func getPodsDataFromYurtHub(url string) ([]byte, error) { // avoid accessing conflict - rand.Seed(time.Now().UnixNano()) time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond) resp, err := http.Get(url) @@ -78,7 +77,7 @@ func getPodsDataFromYurtHub(url string) ([]byte, error) { return nil, fmt.Errorf("could not access yurthub pods API, returned status: %v", resp.Status) } - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if err != nil { return nil, err } diff --git a/pkg/util/iptables/iptables.go b/pkg/util/iptables/iptables.go index 3e92c561ae2..ea7f629a223 100644 --- a/pkg/util/iptables/iptables.go +++ b/pkg/util/iptables/iptables.go @@ -17,11 +17,12 @@ limitations under the License. package iptables import ( + "bufio" "bytes" "context" - "errors" "fmt" "regexp" + "strconv" "strings" "sync" "time" @@ -52,12 +53,17 @@ type Interface interface { FlushChain(table Table, chain Chain) error // DeleteChain deletes the specified chain. If the chain did not exist, return error. DeleteChain(table Table, chain Chain) error + // ChainExists tests whether the specified chain exists, returning an error if it + // does not, or if it is unable to check. + ChainExists(table Table, chain Chain) (bool, error) // EnsureRule checks if the specified rule is present and, if not, creates it. If the rule existed, return true. EnsureRule(position RulePosition, table Table, chain Chain, args ...string) (bool, error) // DeleteRule checks if the specified rule is present and, if so, deletes it. DeleteRule(table Table, chain Chain, args ...string) error - // IsIpv6 returns true if this is managing ipv6 tables - IsIpv6() bool + // IsIPv6 returns true if this is managing ipv6 tables. + IsIPv6() bool + // Protocol returns the IP family this instance is managing, + Protocol() Protocol // SaveInto calls `iptables-save` for table and stores result in a given buffer. SaveInto(table Table, buffer *bytes.Buffer) error // Restore runs `iptables-restore` passing data through []byte. @@ -85,16 +91,19 @@ type Interface interface { // mapped to the same IP:PORT and consequently some suffer packet // drops. HasRandomFully() bool + + // Present checks if the kernel supports the iptable interface + Present() bool } // Protocol defines the ip protocol either ipv4 or ipv6 -type Protocol byte +type Protocol string const ( - // ProtocolIpv4 represents ipv4 protocol in iptables - ProtocolIpv4 Protocol = iota + 1 - // ProtocolIpv6 represents ipv6 protocol in iptables - ProtocolIpv6 + // ProtocolIPv4 represents ipv4 protocol in iptables + ProtocolIPv4 Protocol = "IPv4" + // ProtocolIPv6 represents ipv6 protocol in iptables + ProtocolIPv6 Protocol = "IPv6" ) // Table represents different iptable like filter,nat, mangle and raw @@ -185,9 +194,12 @@ const WaitIntervalString = "-W" // WaitIntervalUsecondsValue a constant for specifying the default wait interval useconds const WaitIntervalUsecondsValue = "100000" -// LockfilePath16x is the iptables lock file acquired by any process that's making any change in the iptable rule +// LockfilePath16x is the iptables 1.6.x lock file acquired by any process that's making any change in the iptable rule const LockfilePath16x = "/run/xtables.lock" +// LockfilePath14x is the iptables 1.4.x lock file acquired by any process that's making any change in the iptable rule +const LockfilePath14x = "@xtables" + // runner implements Interface in terms of exec("iptables"). type runner struct { mu sync.Mutex @@ -197,20 +209,24 @@ type runner struct { hasRandomFully bool waitFlag []string restoreWaitFlag []string - lockfilePath string + lockfilePath14x string + lockfilePath16x string } // newInternal returns a new Interface which will exec iptables, and allows the // caller to change the iptables-restore lockfile path -func newInternal(exec utilexec.Interface, protocol Protocol, lockfilePath string) Interface { +func newInternal(exec utilexec.Interface, protocol Protocol, lockfilePath14x, lockfilePath16x string) Interface { version, err := getIPTablesVersion(exec, protocol) if err != nil { - klog.Warningf("Error checking iptables version, assuming version at least %s: %v", MinCheckVersion, err) + klog.InfoS("Error checking iptables version, assuming version at least", "version", MinCheckVersion, "err", err) version = MinCheckVersion } - if lockfilePath == "" { - lockfilePath = LockfilePath16x + if lockfilePath16x == "" { + lockfilePath16x = LockfilePath16x + } + if lockfilePath14x == "" { + lockfilePath14x = LockfilePath14x } runner := &runner{ @@ -220,14 +236,15 @@ func newInternal(exec utilexec.Interface, protocol Protocol, lockfilePath string hasRandomFully: version.AtLeast(RandomFullyMinVersion), waitFlag: getIPTablesWaitFlag(version), restoreWaitFlag: getIPTablesRestoreWaitFlag(version, exec, protocol), - lockfilePath: lockfilePath, + lockfilePath14x: lockfilePath14x, + lockfilePath16x: lockfilePath16x, } return runner } // New returns a new Interface which will exec iptables. func New(exec utilexec.Interface, protocol Protocol) Interface { - return newInternal(exec, protocol, "") + return newInternal(exec, protocol, "", "") } // EnsureChain is part of Interface. @@ -239,13 +256,12 @@ func (runner *runner) EnsureChain(table Table, chain Chain) (bool, error) { out, err := runner.run(opCreateChain, fullArgs) if err != nil { - var ee utilexec.ExitError - if errors.As(err, &ee) { + if ee, ok := err.(utilexec.ExitError); ok { if ee.Exited() && ee.ExitStatus() == 1 { return true, nil } } - return false, fmt.Errorf("error creating chain %q: %w: %s", chain, err, out) + return false, fmt.Errorf("error creating chain %q: %v: %s", chain, err, out) } return false, nil } @@ -259,7 +275,7 @@ func (runner *runner) FlushChain(table Table, chain Chain) error { out, err := runner.run(opFlushChain, fullArgs) if err != nil { - return fmt.Errorf("error flushing chain %q: %w: %s", chain, err, out) + return fmt.Errorf("error flushing chain %q: %v: %s", chain, err, out) } return nil } @@ -271,10 +287,9 @@ func (runner *runner) DeleteChain(table Table, chain Chain) error { runner.mu.Lock() defer runner.mu.Unlock() - // TODO: we could call iptables -S first, ignore the output and check for non-zero return (more like DeleteRule) out, err := runner.run(opDeleteChain, fullArgs) if err != nil { - return fmt.Errorf("error deleting chain %q: %w: %s", chain, err, out) + return fmt.Errorf("error deleting chain %q: %v: %s", chain, err, out) } return nil } @@ -295,7 +310,7 @@ func (runner *runner) EnsureRule(position RulePosition, table Table, chain Chain } out, err := runner.run(operation(position), fullArgs) if err != nil { - return false, fmt.Errorf("error appending rule: %w: %s", err, out) + return false, fmt.Errorf("error appending rule: %v: %s", err, out) } return false, nil } @@ -316,13 +331,17 @@ func (runner *runner) DeleteRule(table Table, chain Chain, args ...string) error } out, err := runner.run(opDeleteRule, fullArgs) if err != nil { - return fmt.Errorf("error deleting rule: %w: %s", err, out) + return fmt.Errorf("error deleting rule: %v: %s", err, out) } return nil } -func (runner *runner) IsIpv6() bool { - return runner.protocol == ProtocolIpv6 +func (runner *runner) IsIPv6() bool { + return runner.protocol == ProtocolIPv6 +} + +func (runner *runner) Protocol() Protocol { + return runner.protocol } // SaveInto is part of Interface. @@ -336,7 +355,7 @@ func (runner *runner) SaveInto(table Table, buffer *bytes.Buffer) error { // run and return iptablesSaveCmd := iptablesSaveCommand(runner.protocol) args := []string{"-t", string(table)} - klog.V(4).Infof("running %s %v", iptablesSaveCmd, args) + klog.V(4).InfoS("Running", "command", iptablesSaveCmd, "arguments", args) cmd := runner.exec.Command(iptablesSaveCmd, args...) cmd.SetStdout(buffer) stderrBuffer := bytes.NewBuffer(nil) @@ -386,14 +405,14 @@ func (runner *runner) restoreInternal(args []string, data []byte, flush FlushFla // from stepping on each other. iptables-restore 1.6.2 will have // a --wait option like iptables itself, but that's not widely deployed. if len(runner.restoreWaitFlag) == 0 { - locker, err := grabIptablesLocks(runner.lockfilePath) + locker, err := grabIptablesLocks(runner.lockfilePath14x, runner.lockfilePath16x) if err != nil { return err } trace.Step("Locks grabbed") defer func(locker iptablesLocker) { if err := locker.Close(); err != nil { - klog.Errorf("Failed to close iptables locks: %v", err) + klog.ErrorS(err, "Failed to close iptables locks") } }(locker) } @@ -401,25 +420,29 @@ func (runner *runner) restoreInternal(args []string, data []byte, flush FlushFla // run the command and return the output or an error including the output and error fullArgs := append(runner.restoreWaitFlag, args...) iptablesRestoreCmd := iptablesRestoreCommand(runner.protocol) - klog.V(4).Infof("running %s %v", iptablesRestoreCmd, fullArgs) + klog.V(4).InfoS("Running", "command", iptablesRestoreCmd, "arguments", fullArgs) cmd := runner.exec.Command(iptablesRestoreCmd, fullArgs...) cmd.SetStdin(bytes.NewBuffer(data)) b, err := cmd.CombinedOutput() if err != nil { - return fmt.Errorf("%w (%s)", err, b) + pErr, ok := parseRestoreError(string(b)) + if ok { + return pErr + } + return fmt.Errorf("%w: %s", err, b) } return nil } func iptablesSaveCommand(protocol Protocol) string { - if protocol == ProtocolIpv6 { + if protocol == ProtocolIPv6 { return cmdIP6TablesSave } return cmdIPTablesSave } func iptablesRestoreCommand(protocol Protocol) string { - if protocol == ProtocolIpv6 { + if protocol == ProtocolIPv6 { return cmdIP6TablesRestore } return cmdIPTablesRestore @@ -427,7 +450,7 @@ func iptablesRestoreCommand(protocol Protocol) string { } func iptablesCommand(protocol Protocol) string { - if protocol == ProtocolIpv6 { + if protocol == ProtocolIPv6 { return cmdIP6Tables } return cmdIPTables @@ -441,7 +464,7 @@ func (runner *runner) runContext(ctx context.Context, op operation, args []strin iptablesCmd := iptablesCommand(runner.protocol) fullArgs := append(runner.waitFlag, string(op)) fullArgs = append(fullArgs, args...) - klog.V(5).Infof("running iptables: %s %v", iptablesCmd, fullArgs) + klog.V(5).InfoS("Running", "command", iptablesCmd, "arguments", fullArgs) if ctx == nil { return runner.exec.Command(iptablesCmd, fullArgs...).CombinedOutput() } @@ -469,10 +492,10 @@ func trimhex(s string) string { // of hack and half-measures. We should nix this ASAP. func (runner *runner) checkRuleWithoutCheck(table Table, chain Chain, args ...string) (bool, error) { iptablesSaveCmd := iptablesSaveCommand(runner.protocol) - klog.V(1).Infof("running %s -t %s", iptablesSaveCmd, string(table)) + klog.V(1).InfoS("Running", "command", iptablesSaveCmd, "table", string(table)) out, err := runner.exec.Command(iptablesSaveCmd, "-t", string(table)).CombinedOutput() if err != nil { - return false, fmt.Errorf("error checking rule: %w", err) + return false, fmt.Errorf("error checking rule: %v", err) } // Sadly, iptables has inconsistent quoting rules for comments. Just remove all quotes. @@ -508,7 +531,7 @@ func (runner *runner) checkRuleWithoutCheck(table Table, chain Chain, args ...st if sets.NewString(fields...).IsSuperset(argset) { return true, nil } - klog.V(5).Infof("DBG: fields is not a superset of args: fields=%v args=%v", fields, args) + klog.V(5).InfoS("DBG: fields is not a superset of args", "fields", fields, "arguments", args) } return false, nil @@ -520,21 +543,20 @@ func (runner *runner) checkRuleUsingCheck(args []string) (bool, error) { defer cancel() out, err := runner.runContext(ctx, opCheckRule, args) - if errors.Is(ctx.Err(), context.DeadlineExceeded) { + if ctx.Err() == context.DeadlineExceeded { return false, fmt.Errorf("timed out while checking rules") } if err == nil { return true, nil } - var ee utilexec.ExitError - if errors.As(err, &ee) { + if ee, ok := err.(utilexec.ExitError); ok { // iptables uses exit(1) to indicate a failure of the operation, // as compared to a malformed commandline, for example. if ee.Exited() && ee.ExitStatus() == 1 { return false, nil } } - return false, fmt.Errorf("error checking rule: %w: %s", err, out) + return false, fmt.Errorf("error checking rule: %v: %s", err, out) } const ( @@ -547,41 +569,40 @@ const ( // Monitor is part of Interface func (runner *runner) Monitor(canary Chain, tables []Table, reloadFunc func(), interval time.Duration, stopCh <-chan struct{}) { for { - _ = utilwait.PollImmediateUntil(interval, func() (bool, error) { + _ = utilwait.PollUntilContextCancel(context.Background(), interval, true, func(ctx context.Context) (bool, error) { for _, table := range tables { if _, err := runner.EnsureChain(table, canary); err != nil { - klog.Warningf("Could not set up iptables canary %s/%s: %v", string(table), string(canary), err) + klog.ErrorS(err, "Could not set up iptables canary", "table", table, "chain", canary) return false, nil } } return true, nil - }, stopCh) + }) // Poll until stopCh is closed or iptables is flushed - err := utilwait.PollUntil(interval, func() (bool, error) { - if exists, err := runner.chainExists(tables[0], canary); exists { + err := utilwait.PollUntilContextCancel(context.Background(), interval, true, func(ctx context.Context) (bool, error) { + if exists, err := runner.ChainExists(tables[0], canary); exists { return false, nil } else if isResourceError(err) { - klog.Warningf("Could not check for iptables canary %s/%s: %v", string(tables[0]), string(canary), err) + klog.ErrorS(err, "Could not check for iptables canary", "table", tables[0], "chain", canary) return false, nil } - klog.V(2).Infof("iptables canary %s/%s deleted", string(tables[0]), string(canary)) - + klog.V(2).InfoS("IPTables canary deleted", "table", tables[0], "chain", canary) // Wait for the other canaries to be deleted too before returning // so we don't start reloading too soon. - err := utilwait.PollImmediate(iptablesFlushPollTime, iptablesFlushTimeout, func() (bool, error) { + err := utilwait.PollUntilContextTimeout(context.Background(), iptablesFlushPollTime, iptablesFlushTimeout, true, func(ctx context.Context) (bool, error) { for i := 1; i < len(tables); i++ { - if exists, err := runner.chainExists(tables[i], canary); exists || isResourceError(err) { + if exists, err := runner.ChainExists(tables[i], canary); exists || isResourceError(err) { return false, nil } } return true, nil }) if err != nil { - klog.Warning("Inconsistent iptables state detected.") + klog.InfoS("Inconsistent iptables state detected") } return true, nil - }, stopCh) + }) if err != nil { // stopCh was closed @@ -591,19 +612,21 @@ func (runner *runner) Monitor(canary Chain, tables []Table, reloadFunc func(), i return } - klog.V(2).Infof("Reloading after iptables flush") + klog.V(2).InfoS("Reloading after iptables flush") reloadFunc() } } -// chainExists is used internally by Monitor; none of the public Interface methods can be -// used to distinguish "chain exists" from "chain does not exist" with no side effects -func (runner *runner) chainExists(table Table, chain Chain) (bool, error) { +// ChainExists is part of Interface +func (runner *runner) ChainExists(table Table, chain Chain) (bool, error) { fullArgs := makeFullArgs(table, chain) runner.mu.Lock() defer runner.mu.Unlock() + trace := utiltrace.New("iptables ChainExists") + defer trace.LogIfLong(2 * time.Second) + _, err := runner.run(opListChain, fullArgs) return err == nil, err } @@ -614,10 +637,9 @@ const ( opCreateChain operation = "-N" opFlushChain operation = "-F" opDeleteChain operation = "-X" - opListChain operation = "-L" - //opAppendRule operation = "-A" - opCheckRule operation = "-C" - opDeleteRule operation = "-D" + opListChain operation = "-S" + opCheckRule operation = "-C" + opDeleteRule operation = "-D" ) func makeFullArgs(table Table, chain Chain, args ...string) []string { @@ -641,7 +663,7 @@ func getIPTablesVersion(exec utilexec.Interface, protocol Protocol) (*utilversio } version, err := utilversion.ParseGeneric(match[1]) if err != nil { - return nil, fmt.Errorf("iptables version %q is not a valid version string: %w", match[1], err) + return nil, fmt.Errorf("iptables version %q is not a valid version string: %v", match[1], err) } return version, nil @@ -671,11 +693,11 @@ func getIPTablesRestoreWaitFlag(version *utilversion.Version, exec utilexec.Inte // --version, assume it also supports --wait vstring, err := getIPTablesRestoreVersionString(exec, protocol) if err != nil || vstring == "" { - klog.V(3).Infof("couldn't get iptables-restore version; assuming it doesn't support --wait") + klog.V(3).InfoS("Couldn't get iptables-restore version; assuming it doesn't support --wait") return nil } if _, err := utilversion.ParseGeneric(vstring); err != nil { - klog.V(3).Infof("couldn't parse iptables-restore version; assuming it doesn't support --wait") + klog.V(3).InfoS("Couldn't parse iptables-restore version; assuming it doesn't support --wait") return nil } return []string{WaitString} @@ -708,6 +730,16 @@ func (runner *runner) HasRandomFully() bool { return runner.hasRandomFully } +// Present tests if iptable is supported on current kernel by checking the existence +// of default table and chain +func (runner *runner) Present() bool { + if _, err := runner.ChainExists(TableNAT, ChainPostrouting); err != nil { + return false + } + + return true +} + var iptablesNotFoundStrings = []string{ // iptables-legacy [-A|-I] BAD-CHAIN [...] // iptables-legacy [-C|-D] GOOD-CHAIN [...non-matching rule...] @@ -751,9 +783,94 @@ const iptablesStatusResourceProblem = 4 // problem" and was unable to attempt the request. In particular, this will be true if it // times out trying to get the iptables lock. func isResourceError(err error) bool { - var ee utilexec.ExitError - if errors.As(err, &ee) { + if ee, isExitError := err.(utilexec.ExitError); isExitError { return ee.ExitStatus() == iptablesStatusResourceProblem } return false } + +// ParseError records the payload when iptables reports an error parsing its input. +type ParseError interface { + // Line returns the line number on which the parse error was reported. + // NOTE: First line is 1. + Line() int + // Error returns the error message of the parse error, including line number. + Error() string +} + +type parseError struct { + cmd string + line int +} + +func (e parseError) Line() int { + return e.line +} + +func (e parseError) Error() string { + return fmt.Sprintf("%s: input error on line %d: ", e.cmd, e.line) +} + +// LineData represents a single numbered line of data. +type LineData struct { + // Line holds the line number (the first line is 1). + Line int + // The data of the line. + Data string +} + +var regexpParseError = regexp.MustCompile("line ([1-9][0-9]*) failed$") + +// parseRestoreError extracts the line from the error, if it matches returns parseError +// for example: +// input: iptables-restore: line 51 failed +// output: parseError: cmd = iptables-restore, line = 51 +// NOTE: parseRestoreError depends on the error format of iptables, if it ever changes +// we need to update this function +func parseRestoreError(str string) (ParseError, bool) { + errors := strings.Split(str, ":") + if len(errors) != 2 { + return nil, false + } + cmd := errors[0] + matches := regexpParseError.FindStringSubmatch(errors[1]) + if len(matches) != 2 { + return nil, false + } + line, errMsg := strconv.Atoi(matches[1]) + if errMsg != nil { + return nil, false + } + return parseError{cmd: cmd, line: line}, true +} + +// ExtractLines extracts the -count and +count data from the lineNum row of lines and return +// NOTE: lines start from line 1 +func ExtractLines(lines []byte, line, count int) []LineData { + // first line is line 1, so line can't be smaller than 1 + if line < 1 { + return nil + } + start := line - count + if start <= 0 { + start = 1 + } + end := line + count + 1 + + offset := 1 + scanner := bufio.NewScanner(bytes.NewBuffer(lines)) + extractLines := make([]LineData, 0, count*2) + for scanner.Scan() { + if offset >= start && offset < end { + extractLines = append(extractLines, LineData{ + Line: offset, + Data: scanner.Text(), + }) + } + if offset == end { + break + } + offset++ + } + return extractLines +} diff --git a/pkg/util/iptables/iptables_linux.go b/pkg/util/iptables/iptables_linux.go index 237d67acb1f..04c3a49b8d0 100644 --- a/pkg/util/iptables/iptables_linux.go +++ b/pkg/util/iptables/iptables_linux.go @@ -20,6 +20,7 @@ limitations under the License. package iptables import ( + "context" "fmt" "net" "os" @@ -50,7 +51,7 @@ func (l *locker) Close() error { return utilerrors.NewAggregate(errList) } -func grabIptablesLocks(lockfilePath string) (iptablesLocker, error) { +func grabIptablesLocks(lockfilePath14x, lockfilePath16x string) (iptablesLocker, error) { var err error var success bool @@ -67,29 +68,29 @@ func grabIptablesLocks(lockfilePath string) (iptablesLocker, error) { // can't assume which lock method it'll use. // Roughly duplicate iptables 1.6.x xtables_lock() function. - l.lock16, err = os.OpenFile(lockfilePath, os.O_CREATE, 0600) + l.lock16, err = os.OpenFile(lockfilePath16x, os.O_CREATE, 0600) if err != nil { - return nil, fmt.Errorf("failed to open iptables lock %s: %w", lockfilePath, err) + return nil, fmt.Errorf("failed to open iptables lock %s: %v", lockfilePath16x, err) } - if err := wait.PollImmediate(200*time.Millisecond, 2*time.Second, func() (bool, error) { + if err := wait.PollUntilContextTimeout(context.Background(), 200*time.Millisecond, 2*time.Second, true, func(ctx context.Context) (bool, error) { if err := grabIptablesFileLock(l.lock16); err != nil { return false, nil } return true, nil }); err != nil { - return nil, fmt.Errorf("failed to acquire new iptables lock: %w", err) + return nil, fmt.Errorf("failed to acquire new iptables lock: %v", err) } // Roughly duplicate iptables 1.4.x xtables_lock() function. - if err := wait.PollImmediate(200*time.Millisecond, 2*time.Second, func() (bool, error) { - l.lock14, err = net.ListenUnix("unix", &net.UnixAddr{Name: "@xtables", Net: "unix"}) + if err := wait.PollUntilContextTimeout(context.Background(), 200*time.Millisecond, 2*time.Second, true, func(ctx context.Context) (bool, error) { + l.lock14, err = net.ListenUnix("unix", &net.UnixAddr{Name: lockfilePath14x, Net: "unix"}) if err != nil { return false, nil } return true, nil }); err != nil { - return nil, fmt.Errorf("failed to acquire old iptables lock: %w", err) + return nil, fmt.Errorf("failed to acquire old iptables lock: %v", err) } success = true diff --git a/pkg/util/iptables/iptables_test.go b/pkg/util/iptables/iptables_test.go index 416a0de1885..f97fe384d52 100644 --- a/pkg/util/iptables/iptables_test.go +++ b/pkg/util/iptables/iptables_test.go @@ -21,33 +21,32 @@ package iptables import ( "bytes" + "context" "fmt" "net" "os" "reflect" "strings" "testing" + "time" "k8s.io/apimachinery/pkg/util/sets" utilversion "k8s.io/apimachinery/pkg/util/version" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/utils/exec" fakeexec "k8s.io/utils/exec/testing" ) -const TestLockfilePath = "xtables.lock" - -func protocolStr(protocol Protocol) string { - if protocol == ProtocolIpv4 { - return "IPv4" - } - return "IPv6" +func getLockPaths() (string, string) { + lock14x := fmt.Sprintf("@xtables-%d", time.Now().Nanosecond()) + lock16x := fmt.Sprintf("xtables-%d.lock", time.Now().Nanosecond()) + return lock14x, lock16x } func testIPTablesVersionCmds(t *testing.T, protocol Protocol) { version := " v1.4.22" iptablesCmd := iptablesCommand(protocol) iptablesRestoreCmd := iptablesRestoreCommand(protocol) - protoStr := protocolStr(protocol) fcmd := fakeexec.FakeCmd{ CombinedOutputScript: []fakeexec.FakeAction{ @@ -57,36 +56,34 @@ func testIPTablesVersionCmds(t *testing.T, protocol Protocol) { func() ([]byte, []byte, error) { return []byte(iptablesRestoreCmd + version), nil, nil }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - _ = New(&fexec, protocol) + _ = New(fexec, protocol) // Check that proper iptables version command was used during runner instantiation if !sets.NewString(fcmd.CombinedOutputLog[0]...).HasAll(iptablesCmd, "--version") { - t.Errorf("%s runner instantiate: Expected cmd '%s --version', Got '%s'", protoStr, iptablesCmd, fcmd.CombinedOutputLog[0]) + t.Errorf("%s runner instantiate: Expected cmd '%s --version', Got '%s'", protocol, iptablesCmd, fcmd.CombinedOutputLog[0]) } // Check that proper iptables restore version command was used during runner instantiation if !sets.NewString(fcmd.CombinedOutputLog[1]...).HasAll(iptablesRestoreCmd, "--version") { - t.Errorf("%s runner instantiate: Expected cmd '%s --version', Got '%s'", protoStr, iptablesRestoreCmd, fcmd.CombinedOutputLog[1]) + t.Errorf("%s runner instantiate: Expected cmd '%s --version', Got '%s'", protocol, iptablesRestoreCmd, fcmd.CombinedOutputLog[1]) } } func TestIPTablesVersionCmdsIPv4(t *testing.T) { - testIPTablesVersionCmds(t, ProtocolIpv4) + testIPTablesVersionCmds(t, ProtocolIPv4) } func TestIPTablesVersionCmdsIPv6(t *testing.T) { - testIPTablesVersionCmds(t, ProtocolIpv6) + testIPTablesVersionCmds(t, ProtocolIPv6) } func testEnsureChain(t *testing.T, protocol Protocol) { - protoStr := protocolStr(protocol) - fcmd := fakeexec.FakeCmd{ CombinedOutputScript: []fakeexec.FakeAction{ // iptables version check @@ -99,7 +96,7 @@ func testEnsureChain(t *testing.T, protocol Protocol) { func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 2} }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, @@ -107,43 +104,43 @@ func testEnsureChain(t *testing.T, protocol Protocol) { func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := New(&fexec, protocol) + runner := New(fexec, protocol) // Success. exists, err := runner.EnsureChain(TableNAT, Chain("FOOBAR")) if err != nil { - t.Errorf("%s new chain: Expected success, got %v", protoStr, err) + t.Errorf("%s new chain: Expected success, got %v", protocol, err) } if exists { - t.Errorf("%s new chain: Expected exists = false", protoStr) + t.Errorf("%s new chain: Expected exists = false", protocol) } if fcmd.CombinedOutputCalls != 2 { - t.Errorf("%s new chain: Expected 2 CombinedOutput() calls, got %d", protoStr, fcmd.CombinedOutputCalls) + t.Errorf("%s new chain: Expected 2 CombinedOutput() calls, got %d", protocol, fcmd.CombinedOutputCalls) } cmd := iptablesCommand(protocol) if !sets.NewString(fcmd.CombinedOutputLog[1]...).HasAll(cmd, "-t", "nat", "-N", "FOOBAR") { - t.Errorf("%s new chain: Expected cmd containing '%s -t nat -N FOOBAR', got %s", protoStr, cmd, fcmd.CombinedOutputLog[2]) + t.Errorf("%s new chain: Expected cmd containing '%s -t nat -N FOOBAR', got %s", protocol, cmd, fcmd.CombinedOutputLog[2]) } // Exists. exists, err = runner.EnsureChain(TableNAT, Chain("FOOBAR")) if err != nil { - t.Errorf("%s existing chain: Expected success, got %v", protoStr, err) + t.Errorf("%s existing chain: Expected success, got %v", protocol, err) } if !exists { - t.Errorf("%s existing chain: Expected exists = true", protoStr) + t.Errorf("%s existing chain: Expected exists = true", protocol) } // Simulate failure. _, err = runner.EnsureChain(TableNAT, Chain("FOOBAR")) if err == nil { - t.Errorf("%s: Expected failure", protoStr) + t.Errorf("%s: Expected failure", protocol) } } -func TestEnsureChainIpv4(t *testing.T) { - testEnsureChain(t, ProtocolIpv4) +func TestEnsureChainIPv4(t *testing.T) { + testEnsureChain(t, ProtocolIPv4) } -func TestEnsureChainIpv6(t *testing.T) { - testEnsureChain(t, ProtocolIpv6) +func TestEnsureChainIPv6(t *testing.T) { + testEnsureChain(t, ProtocolIPv6) } func TestFlushChain(t *testing.T) { @@ -157,14 +154,14 @@ func TestFlushChain(t *testing.T) { func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := New(&fexec, ProtocolIpv4) + runner := New(fexec, ProtocolIPv4) // Success. err := runner.FlushChain(TableNAT, Chain("FOOBAR")) if err != nil { @@ -194,14 +191,14 @@ func TestDeleteChain(t *testing.T) { func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := New(&fexec, ProtocolIpv4) + runner := New(fexec, ProtocolIPv4) // Success. err := runner.DeleteChain(TableNAT, Chain("FOOBAR")) if err != nil { @@ -229,7 +226,7 @@ func TestEnsureRuleAlreadyExists(t *testing.T) { func() ([]byte, []byte, error) { return []byte{}, nil, nil }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ // iptables version check func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, @@ -237,7 +234,7 @@ func TestEnsureRuleAlreadyExists(t *testing.T) { func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := New(&fexec, ProtocolIpv4) + runner := New(fexec, ProtocolIPv4) exists, err := runner.EnsureRule(Append, TableNAT, ChainOutput, "abc", "123") if err != nil { t.Errorf("expected success, got %v", err) @@ -264,7 +261,7 @@ func TestEnsureRuleNew(t *testing.T) { func() ([]byte, []byte, error) { return []byte{}, nil, nil }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ // iptables version check func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, @@ -273,7 +270,7 @@ func TestEnsureRuleNew(t *testing.T) { func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := New(&fexec, ProtocolIpv4) + runner := New(fexec, ProtocolIPv4) exists, err := runner.EnsureRule(Append, TableNAT, ChainOutput, "abc", "123") if err != nil { t.Errorf("expected success, got %v", err) @@ -298,7 +295,7 @@ func TestEnsureRuleErrorChecking(t *testing.T) { func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 2} }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ // iptables version check func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, @@ -306,7 +303,7 @@ func TestEnsureRuleErrorChecking(t *testing.T) { func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := New(&fexec, ProtocolIpv4) + runner := New(fexec, ProtocolIPv4) _, err := runner.EnsureRule(Append, TableNAT, ChainOutput, "abc", "123") if err == nil { t.Errorf("expected failure") @@ -327,7 +324,7 @@ func TestEnsureRuleErrorCreating(t *testing.T) { func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ // iptables version check func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, @@ -336,7 +333,7 @@ func TestEnsureRuleErrorCreating(t *testing.T) { func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := New(&fexec, ProtocolIpv4) + runner := New(fexec, ProtocolIPv4) _, err := runner.EnsureRule(Append, TableNAT, ChainOutput, "abc", "123") if err == nil { t.Errorf("expected failure") @@ -355,7 +352,7 @@ func TestDeleteRuleDoesNotExist(t *testing.T) { func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ // iptables version check func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, @@ -363,7 +360,7 @@ func TestDeleteRuleDoesNotExist(t *testing.T) { func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := New(&fexec, ProtocolIpv4) + runner := New(fexec, ProtocolIPv4) err := runner.DeleteRule(TableNAT, ChainOutput, "abc", "123") if err != nil { t.Errorf("expected success, got %v", err) @@ -387,7 +384,7 @@ func TestDeleteRuleExists(t *testing.T) { func() ([]byte, []byte, error) { return []byte{}, nil, nil }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ // iptables version check func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, @@ -396,7 +393,7 @@ func TestDeleteRuleExists(t *testing.T) { func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := New(&fexec, ProtocolIpv4) + runner := New(fexec, ProtocolIPv4) err := runner.DeleteRule(TableNAT, ChainOutput, "abc", "123") if err != nil { t.Errorf("expected success, got %v", err) @@ -418,7 +415,7 @@ func TestDeleteRuleErrorChecking(t *testing.T) { func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 2} }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ // iptables version check func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, @@ -426,7 +423,7 @@ func TestDeleteRuleErrorChecking(t *testing.T) { func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := New(&fexec, ProtocolIpv4) + runner := New(fexec, ProtocolIPv4) err := runner.DeleteRule(TableNAT, ChainOutput, "abc", "123") if err == nil { t.Errorf("expected failure") @@ -447,7 +444,7 @@ func TestDeleteRuleErrorDeleting(t *testing.T) { func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ // iptables version check func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, @@ -456,7 +453,7 @@ func TestDeleteRuleErrorDeleting(t *testing.T) { func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := New(&fexec, ProtocolIpv4) + runner := New(fexec, ProtocolIPv4) err := runner.DeleteRule(TableNAT, ChainOutput, "abc", "123") if err == nil { t.Errorf("expected failure") @@ -485,13 +482,13 @@ func TestGetIPTablesHasCheckCommand(t *testing.T) { func() ([]byte, []byte, error) { return []byte(testCase.Version), nil, nil }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - ipt := New(&fexec, ProtocolIpv4) + ipt := New(fexec, ProtocolIPv4) runner := ipt.(*runner) if testCase.Expected != runner.hasCheck { t.Errorf("Expected result: %v, Got result: %v", testCase.Expected, runner.hasCheck) @@ -505,12 +502,12 @@ func TestIPTablesCommands(t *testing.T) { protocol Protocol expectedCmd string }{ - {"iptablesCommand", ProtocolIpv4, cmdIPTables}, - {"iptablesCommand", ProtocolIpv6, cmdIP6Tables}, - {"iptablesSaveCommand", ProtocolIpv4, cmdIPTablesSave}, - {"iptablesSaveCommand", ProtocolIpv6, cmdIP6TablesSave}, - {"iptablesRestoreCommand", ProtocolIpv4, cmdIPTablesRestore}, - {"iptablesRestoreCommand", ProtocolIpv6, cmdIP6TablesRestore}, + {"iptablesCommand", ProtocolIPv4, cmdIPTables}, + {"iptablesCommand", ProtocolIPv6, cmdIP6Tables}, + {"iptablesSaveCommand", ProtocolIPv4, cmdIPTablesSave}, + {"iptablesSaveCommand", ProtocolIPv6, cmdIP6TablesSave}, + {"iptablesRestoreCommand", ProtocolIPv4, cmdIPTablesRestore}, + {"iptablesRestoreCommand", ProtocolIPv6, cmdIP6TablesRestore}, } for _, testCase := range testCases { var cmd string @@ -544,13 +541,13 @@ COMMIT func() ([]byte, []byte, error) { return []byte(iptablesSaveOutput), nil, nil }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ // The first Command() call is checking the rule. Success of that exec means "done". func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := &runner{exec: &fexec} + runner := &runner{exec: fexec} exists, err := runner.checkRuleWithoutCheck( TableNAT, ChainPrerouting, "-m", "addrtype", @@ -587,13 +584,13 @@ COMMIT func() ([]byte, []byte, error) { return []byte(iptablesSaveOutput), nil, nil }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ // The first Command() call is checking the rule. Success of that exec means "done". func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := &runner{exec: &fexec} + runner := &runner{exec: fexec} exists, err := runner.checkRuleWithoutCheck(TableNAT, ChainPrerouting, "-m", "addrtype", "-j", "DOCKER") if err != nil { t.Errorf("expected success, got %v", err) @@ -643,7 +640,7 @@ func TestWaitFlagUnavailable(t *testing.T) { func() ([]byte, []byte, error) { return []byte{}, nil, nil }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ // iptables version check func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, @@ -652,7 +649,7 @@ func TestWaitFlagUnavailable(t *testing.T) { func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := New(&fexec, ProtocolIpv4) + runner := New(fexec, ProtocolIPv4) err := runner.DeleteChain(TableNAT, Chain("FOOBAR")) if err != nil { t.Errorf("expected success, got %v", err) @@ -676,14 +673,14 @@ func TestWaitFlagOld(t *testing.T) { func() ([]byte, []byte, error) { return []byte{}, nil, nil }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := New(&fexec, ProtocolIpv4) + runner := New(fexec, ProtocolIPv4) err := runner.DeleteChain(TableNAT, Chain("FOOBAR")) if err != nil { t.Errorf("expected success, got %v", err) @@ -710,14 +707,14 @@ func TestWaitFlagNew(t *testing.T) { func() ([]byte, []byte, error) { return []byte{}, nil, nil }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := New(&fexec, ProtocolIpv4) + runner := New(fexec, ProtocolIPv4) err := runner.DeleteChain(TableNAT, Chain("FOOBAR")) if err != nil { t.Errorf("expected success, got %v", err) @@ -741,14 +738,14 @@ func TestWaitIntervalFlagNew(t *testing.T) { func() ([]byte, []byte, error) { return []byte{}, nil, nil }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := New(&fexec, ProtocolIpv4) + runner := New(fexec, ProtocolIPv4) err := runner.DeleteChain(TableNAT, Chain("FOOBAR")) if err != nil { t.Errorf("expected success, got %v", err) @@ -765,7 +762,6 @@ func testSaveInto(t *testing.T, protocol Protocol) { version := " v1.9.22" iptablesCmd := iptablesCommand(protocol) iptablesSaveCmd := iptablesSaveCommand(protocol) - protoStr := protocolStr(protocol) output := fmt.Sprintf(`# Generated by %s on Thu Jan 19 11:38:09 2017 *filter @@ -787,60 +783,59 @@ COMMIT func() ([]byte, []byte, error) { return nil, []byte(stderrOutput), &fakeexec.FakeExitError{Status: 1} }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := New(&fexec, protocol) + runner := New(fexec, protocol) buffer := bytes.NewBuffer(nil) // Success. err := runner.SaveInto(TableNAT, buffer) if err != nil { - t.Fatalf("%s: Expected success, got %v", protoStr, err) + t.Fatalf("%s: Expected success, got %v", protocol, err) } - if string(buffer.Bytes()) != output { - t.Errorf("%s: Expected output '%s', got '%v'", protoStr, output, string(buffer.Bytes())) + if buffer.String() != output { + t.Errorf("%s: Expected output '%s', got '%v'", protocol, output, buffer.String()) } if fcmd.CombinedOutputCalls != 1 { - t.Errorf("%s: Expected 1 CombinedOutput() calls, got %d", protoStr, fcmd.CombinedOutputCalls) + t.Errorf("%s: Expected 1 CombinedOutput() calls, got %d", protocol, fcmd.CombinedOutputCalls) } if fcmd.RunCalls != 1 { - t.Errorf("%s: Expected 1 Run() call, got %d", protoStr, fcmd.RunCalls) + t.Errorf("%s: Expected 1 Run() call, got %d", protocol, fcmd.RunCalls) } if !sets.NewString(fcmd.RunLog[0]...).HasAll(iptablesSaveCmd, "-t", "nat") { - t.Errorf("%s: Expected cmd containing '%s -t nat', got '%s'", protoStr, iptablesSaveCmd, fcmd.RunLog[0]) + t.Errorf("%s: Expected cmd containing '%s -t nat', got '%s'", protocol, iptablesSaveCmd, fcmd.RunLog[0]) } // Failure. buffer.Reset() err = runner.SaveInto(TableNAT, buffer) if err == nil { - t.Errorf("%s: Expected failure", protoStr) + t.Errorf("%s: Expected failure", protocol) } - if string(buffer.Bytes()) != stderrOutput { - t.Errorf("%s: Expected output '%s', got '%v'", protoStr, stderrOutput, string(buffer.Bytes())) + if buffer.String() != stderrOutput { + t.Errorf("%s: Expected output '%s', got '%v'", protocol, stderrOutput, buffer.String()) } } func TestSaveIntoIPv4(t *testing.T) { - testSaveInto(t, ProtocolIpv4) + testSaveInto(t, ProtocolIPv4) } func TestSaveIntoIPv6(t *testing.T) { - testSaveInto(t, ProtocolIpv6) + testSaveInto(t, ProtocolIPv6) } func testRestore(t *testing.T, protocol Protocol) { version := " v1.9.22" iptablesCmd := iptablesCommand(protocol) iptablesRestoreCmd := iptablesRestoreCommand(protocol) - protoStr := protocolStr(protocol) fcmd := fakeexec.FakeCmd{ CombinedOutputScript: []fakeexec.FakeAction{ @@ -853,7 +848,7 @@ func testRestore(t *testing.T, protocol Protocol) { func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, @@ -863,69 +858,69 @@ func testRestore(t *testing.T, protocol Protocol) { func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := New(&fexec, protocol) + runner := New(fexec, protocol) // both flags true err := runner.Restore(TableNAT, []byte{}, FlushTables, RestoreCounters) if err != nil { - t.Errorf("%s flush,restore: Expected success, got %v", protoStr, err) + t.Errorf("%s flush,restore: Expected success, got %v", protocol, err) } commandSet := sets.NewString(fcmd.CombinedOutputLog[1]...) if !commandSet.HasAll(iptablesRestoreCmd, "-T", string(TableNAT), "--counters") || commandSet.HasAny("--noflush") { - t.Errorf("%s flush, restore: Expected cmd containing '%s -T %s --counters', got '%s'", protoStr, iptablesRestoreCmd, string(TableNAT), fcmd.CombinedOutputLog[1]) + t.Errorf("%s flush, restore: Expected cmd containing '%s -T %s --counters', got '%s'", protocol, iptablesRestoreCmd, string(TableNAT), fcmd.CombinedOutputLog[1]) } // FlushTables, NoRestoreCounters err = runner.Restore(TableNAT, []byte{}, FlushTables, NoRestoreCounters) if err != nil { - t.Errorf("%s flush, no restore: Expected success, got %v", protoStr, err) + t.Errorf("%s flush, no restore: Expected success, got %v", protocol, err) } commandSet = sets.NewString(fcmd.CombinedOutputLog[2]...) if !commandSet.HasAll(iptablesRestoreCmd, "-T", string(TableNAT)) || commandSet.HasAny("--noflush", "--counters") { - t.Errorf("%s flush, no restore: Expected cmd containing '--noflush' or '--counters', got '%s'", protoStr, fcmd.CombinedOutputLog[2]) + t.Errorf("%s flush, no restore: Expected cmd containing '--noflush' or '--counters', got '%s'", protocol, fcmd.CombinedOutputLog[2]) } // NoFlushTables, RestoreCounters err = runner.Restore(TableNAT, []byte{}, NoFlushTables, RestoreCounters) if err != nil { - t.Errorf("%s no flush, restore: Expected success, got %v", protoStr, err) + t.Errorf("%s no flush, restore: Expected success, got %v", protocol, err) } commandSet = sets.NewString(fcmd.CombinedOutputLog[3]...) if !commandSet.HasAll(iptablesRestoreCmd, "-T", string(TableNAT), "--noflush", "--counters") { - t.Errorf("%s no flush, restore: Expected cmd containing '--noflush' and '--counters', got '%s'", protoStr, fcmd.CombinedOutputLog[3]) + t.Errorf("%s no flush, restore: Expected cmd containing '--noflush' and '--counters', got '%s'", protocol, fcmd.CombinedOutputLog[3]) } // NoFlushTables, NoRestoreCounters err = runner.Restore(TableNAT, []byte{}, NoFlushTables, NoRestoreCounters) if err != nil { - t.Errorf("%s no flush, no restore: Expected success, got %v", protoStr, err) + t.Errorf("%s no flush, no restore: Expected success, got %v", protocol, err) } commandSet = sets.NewString(fcmd.CombinedOutputLog[4]...) if !commandSet.HasAll(iptablesRestoreCmd, "-T", string(TableNAT), "--noflush") || commandSet.HasAny("--counters") { - t.Errorf("%s no flush, no restore: Expected cmd containing '%s -T %s --noflush', got '%s'", protoStr, iptablesRestoreCmd, string(TableNAT), fcmd.CombinedOutputLog[4]) + t.Errorf("%s no flush, no restore: Expected cmd containing '%s -T %s --noflush', got '%s'", protocol, iptablesRestoreCmd, string(TableNAT), fcmd.CombinedOutputLog[4]) } if fcmd.CombinedOutputCalls != 5 { - t.Errorf("%s: Expected 5 total CombinedOutput() calls, got %d", protoStr, fcmd.CombinedOutputCalls) + t.Errorf("%s: Expected 5 total CombinedOutput() calls, got %d", protocol, fcmd.CombinedOutputCalls) } // Failure. err = runner.Restore(TableNAT, []byte{}, FlushTables, RestoreCounters) if err == nil { - t.Errorf("%s Expected a failure", protoStr) + t.Errorf("%s Expected a failure", protocol) } } func TestRestoreIPv4(t *testing.T) { - testRestore(t, ProtocolIpv4) + testRestore(t, ProtocolIPv4) } func TestRestoreIPv6(t *testing.T) { - testRestore(t, ProtocolIpv6) + testRestore(t, ProtocolIPv6) } // TestRestoreAll tests only the simplest use case, as flag handling code is already tested in TestRestore @@ -938,15 +933,15 @@ func TestRestoreAll(t *testing.T) { func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := newInternal(&fexec, ProtocolIpv4, TestLockfilePath) - defer os.Remove(TestLockfilePath) + lockPath14x, lockPath16x := getLockPaths() + runner := newInternal(fexec, ProtocolIPv4, lockPath14x, lockPath16x) err := runner.RestoreAll([]byte{}, NoFlushTables, RestoreCounters) if err != nil { @@ -979,15 +974,15 @@ func TestRestoreAllWait(t *testing.T) { func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := newInternal(&fexec, ProtocolIpv4, TestLockfilePath) - defer os.Remove(TestLockfilePath) + lockPath14x, lockPath16x := getLockPaths() + runner := newInternal(fexec, ProtocolIPv4, lockPath14x, lockPath16x) err := runner.RestoreAll([]byte{}, NoFlushTables, RestoreCounters) if err != nil { @@ -1023,7 +1018,7 @@ func TestRestoreAllWaitOldIptablesRestore(t *testing.T) { func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, @@ -1031,8 +1026,11 @@ func TestRestoreAllWaitOldIptablesRestore(t *testing.T) { func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := newInternal(&fexec, ProtocolIpv4, TestLockfilePath) - defer os.Remove(TestLockfilePath) + lockPath14x, lockPath16x := getLockPaths() + // the lockPath14x is a UNIX socket which is cleaned up automatically on close, but the + // lockPath16x is a plain file which is not cleaned up. + defer os.Remove(lockPath16x) + runner := newInternal(fexec, ProtocolIPv4, lockPath14x, lockPath16x) err := runner.RestoreAll([]byte{}, NoFlushTables, RestoreCounters) if err != nil { @@ -1070,32 +1068,34 @@ func TestRestoreAllGrabNewLock(t *testing.T) { func() ([]byte, []byte, error) { return []byte{}, nil, nil }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - - runner := newInternal(&fexec, ProtocolIpv4, TestLockfilePath) - defer os.Remove(TestLockfilePath) + lockPath14x, lockPath16x := getLockPaths() + runner := newInternal(fexec, ProtocolIPv4, lockPath14x, lockPath16x) // Grab the /run lock and ensure the RestoreAll fails - runLock, err := os.OpenFile(TestLockfilePath, os.O_CREATE, 0600) + runLock, err := os.OpenFile(lockPath16x, os.O_CREATE, 0600) if err != nil { - t.Fatalf("expected to open %s, got %v", TestLockfilePath, err) + t.Fatalf("expected to open %s, got %v", lockPath16x, err) } - defer runLock.Close() + defer func() { + runLock.Close() + os.Remove(lockPath16x) + }() if err := grabIptablesFileLock(runLock); err != nil { - t.Errorf("expected to lock %s, got %v", TestLockfilePath, err) + t.Errorf("expected to lock %s, got %v", lockPath16x, err) } err = runner.RestoreAll([]byte{}, NoFlushTables, RestoreCounters) if err == nil { - t.Errorf("expected failure, got success instead") + t.Fatal("expected failure, got success instead") } - if !strings.Contains(err.Error(), "failed to acquire new iptables lock: timed out waiting for the condition") { + if !strings.Contains(err.Error(), "failed to acquire new iptables lock: context deadline exceeded") { t.Errorf("expected timeout error, got %v", err) } } @@ -1112,28 +1112,42 @@ func TestRestoreAllGrabOldLock(t *testing.T) { func() ([]byte, []byte, error) { return []byte{}, nil, nil }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - - runner := newInternal(&fexec, ProtocolIpv4, TestLockfilePath) - defer os.Remove(TestLockfilePath) - - // Grab the abstract @xtables socket - runLock, err := net.ListenUnix("unix", &net.UnixAddr{Name: "@xtables", Net: "unix"}) + lockPath14x, lockPath16x := getLockPaths() + // the lockPath14x is a UNIX socket which is cleaned up automatically on close, but the + // lockPath16x is a plain file which is not cleaned up. + defer os.Remove(lockPath16x) + runner := newInternal(fexec, ProtocolIPv4, lockPath14x, lockPath16x) + + var runLock *net.UnixListener + // Grab the abstract @xtables socket, will retry if the socket exists + err := wait.PollUntilContextTimeout(context.Background(), time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (done bool, err error) { + runLock, err = net.ListenUnix("unix", &net.UnixAddr{Name: lockPath14x, Net: "unix"}) + if err != nil { + t.Logf("Failed to lock %s: %v, will retry.", lockPath14x, err) + return false, nil + } + return true, nil + }) if err != nil { - t.Fatalf("expected to lock @xtables, got %v", err) + t.Fatalf("Timed out locking %s", lockPath14x) + } + if runLock == nil { + t.Fatal("Unexpected nil runLock") } + defer runLock.Close() err = runner.RestoreAll([]byte{}, NoFlushTables, RestoreCounters) if err == nil { - t.Errorf("expected failure, got success instead") + t.Fatal("expected failure, got success instead") } - if !strings.Contains(err.Error(), "failed to acquire old iptables lock: timed out waiting for the condition") { + if !strings.Contains(err.Error(), "failed to acquire old iptables lock: context deadline exceeded") { t.Errorf("expected timeout error, got %v", err) } } @@ -1151,7 +1165,7 @@ func TestRestoreAllWaitBackportedIptablesRestore(t *testing.T) { func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, }, } - fexec := fakeexec.FakeExec{ + fexec := &fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, @@ -1159,8 +1173,8 @@ func TestRestoreAllWaitBackportedIptablesRestore(t *testing.T) { func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } - runner := newInternal(&fexec, ProtocolIpv4, TestLockfilePath) - defer os.Remove(TestLockfilePath) + lockPath14x, lockPath16x := getLockPaths() + runner := newInternal(fexec, ProtocolIPv4, lockPath14x, lockPath16x) err := runner.RestoreAll([]byte{}, NoFlushTables, RestoreCounters) if err != nil { @@ -1185,3 +1199,63 @@ func TestRestoreAllWaitBackportedIptablesRestore(t *testing.T) { t.Errorf("expected failure") } } + +// TestExtractLines tests that +func TestExtractLines(t *testing.T) { + mkLines := func(lines ...LineData) []LineData { + return lines + } + lines := "Line1: 1\nLine2: 2\nLine3: 3\nLine4: 4\nLine5: 5\nLine6: 6\nLine7: 7\nLine8: 8\nLine9: 9\nLine10: 10" + tests := []struct { + count int + line int + name string + want []LineData + }{{ + name: "test-line-0", + count: 3, + line: 0, + want: nil, + }, { + name: "test-count-0", + count: 0, + line: 3, + want: mkLines(LineData{3, "Line3: 3"}), + }, { + name: "test-common-cases", + count: 3, + line: 6, + want: mkLines( + LineData{3, "Line3: 3"}, + LineData{4, "Line4: 4"}, + LineData{5, "Line5: 5"}, + LineData{6, "Line6: 6"}, + LineData{7, "Line7: 7"}, + LineData{8, "Line8: 8"}, + LineData{9, "Line9: 9"}), + }, { + name: "test4-bound-cases", + count: 11, + line: 10, + want: mkLines( + LineData{1, "Line1: 1"}, + LineData{2, "Line2: 2"}, + LineData{3, "Line3: 3"}, + LineData{4, "Line4: 4"}, + LineData{5, "Line5: 5"}, + LineData{6, "Line6: 6"}, + LineData{7, "Line7: 7"}, + LineData{8, "Line8: 8"}, + LineData{9, "Line9: 9"}, + LineData{10, "Line10: 10"}), + }} + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ExtractLines([]byte(lines), tt.line, tt.count) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("got = %v, want = %v", got, tt.want) + } + }) + } +} diff --git a/pkg/util/iptables/iptables_unsupported.go b/pkg/util/iptables/iptables_unsupported.go index a11f936cf51..f7be7ed8dfd 100644 --- a/pkg/util/iptables/iptables_unsupported.go +++ b/pkg/util/iptables/iptables_unsupported.go @@ -21,13 +21,8 @@ package iptables import ( "fmt" - "os" ) -func grabIptablesLocks(lockfilePath string) (iptablesLocker, error) { +func grabIptablesLocks(lock14filePath, lock16filePath string) (iptablesLocker, error) { return nil, fmt.Errorf("iptables unsupported on this platform") } - -func grabIptablesFileLock(f *os.File) error { - return fmt.Errorf("iptables unsupported on this platform") -} diff --git a/pkg/util/iptables/iptables_unsupported_test.go b/pkg/util/iptables/iptables_unsupported_test.go deleted file mode 100644 index 6691d3147f5..00000000000 --- a/pkg/util/iptables/iptables_unsupported_test.go +++ /dev/null @@ -1,35 +0,0 @@ -/* -Copyright 2021 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package iptables - -import ( - "testing" -) - -func Test_grabIptablesFileLock(t *testing.T) { - err := grabIptablesFileLock(nil) - if err != nil { - t.Log(err.Error()) - } -} - -func Test_grabIptablesLocks(t *testing.T) { - err := grabIptablesFileLock(nil) - if err != nil { - t.Log(err.Error()) - } -} diff --git a/pkg/util/iptables/monitor_test.go b/pkg/util/iptables/monitor_test.go deleted file mode 100644 index 16f911ffeaa..00000000000 --- a/pkg/util/iptables/monitor_test.go +++ /dev/null @@ -1,338 +0,0 @@ -//go:build linux -// +build linux - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package iptables - -import ( - "context" - "fmt" - "io" - "sync" - "sync/atomic" - "testing" - "time" - - "k8s.io/apimachinery/pkg/util/sets" - utilwait "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/utils/exec" -) - -// We can't use the normal FakeExec because we don't know precisely how many times the -// Monitor thread will do its checks, and we don't know precisely how its iptables calls -// will interleave with the main thread's. So we use our own fake Exec implementation that -// implements a minimal iptables interface. This will need updates as iptables.runner -// changes its use of Exec. -type monitorFakeExec struct { - sync.Mutex - - tables map[string]sets.String - - block bool - wasBlocked bool -} - -func newMonitorFakeExec() *monitorFakeExec { - tables := make(map[string]sets.String) - tables["mangle"] = sets.NewString() - tables["filter"] = sets.NewString() - tables["nat"] = sets.NewString() - return &monitorFakeExec{tables: tables} -} - -func (mfe *monitorFakeExec) blockIPTables(block bool) { - mfe.Lock() - defer mfe.Unlock() - - mfe.block = block -} - -func (mfe *monitorFakeExec) getWasBlocked() bool { - mfe.Lock() - defer mfe.Unlock() - - wasBlocked := mfe.wasBlocked - mfe.wasBlocked = false - return wasBlocked -} - -func (mfe *monitorFakeExec) Command(cmd string, args ...string) exec.Cmd { - return &monitorFakeCmd{mfe: mfe, cmd: cmd, args: args} -} - -func (mfe *monitorFakeExec) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd { - return mfe.Command(cmd, args...) -} - -func (mfe *monitorFakeExec) LookPath(file string) (string, error) { - return file, nil -} - -type monitorFakeCmd struct { - mfe *monitorFakeExec - cmd string - args []string -} - -func (mfc *monitorFakeCmd) CombinedOutput() ([]byte, error) { - if mfc.cmd == cmdIPTablesRestore { - // Only used for "iptables-restore --version", and the result doesn't matter - return []byte{}, nil - } else if mfc.cmd != cmdIPTables { - panic("bad command " + mfc.cmd) - } - - if len(mfc.args) == 1 && mfc.args[0] == "--version" { - return []byte("iptables v1.6.2"), nil - } - - if len(mfc.args) != 8 || mfc.args[0] != WaitString || mfc.args[1] != WaitSecondsValue || mfc.args[2] != WaitIntervalString || mfc.args[3] != WaitIntervalUsecondsValue || mfc.args[6] != "-t" { - panic(fmt.Sprintf("bad args %#v", mfc.args)) - } - op := operation(mfc.args[4]) - chainName := mfc.args[5] - tableName := mfc.args[7] - - mfc.mfe.Lock() - defer mfc.mfe.Unlock() - - table := mfc.mfe.tables[tableName] - if table == nil { - return []byte{}, fmt.Errorf("no such table %q", tableName) - } - - // For ease-of-testing reasons, blockIPTables blocks create and list, but not delete - if mfc.mfe.block && op != opDeleteChain { - mfc.mfe.wasBlocked = true - return []byte{}, exec.CodeExitError{Code: 4, Err: fmt.Errorf("could not get xtables.lock, etc")} - } - - switch op { - case opCreateChain: - if !table.Has(chainName) { - table.Insert(chainName) - } - return []byte{}, nil - case opListChain: - if table.Has(chainName) { - return []byte{}, nil - } - return []byte{}, fmt.Errorf("no such chain %q", chainName) - case opDeleteChain: - table.Delete(chainName) - return []byte{}, nil - default: - panic("should not be reached") - } -} - -func (mfc *monitorFakeCmd) SetStdin(in io.Reader) { - // Used by getIPTablesRestoreVersionString(), can be ignored -} - -func (mfc *monitorFakeCmd) Run() error { - panic("should not be reached") -} -func (mfc *monitorFakeCmd) Output() ([]byte, error) { - panic("should not be reached") -} -func (mfc *monitorFakeCmd) SetDir(dir string) { - panic("should not be reached") -} -func (mfc *monitorFakeCmd) SetStdout(out io.Writer) { - panic("should not be reached") -} -func (mfc *monitorFakeCmd) SetStderr(out io.Writer) { - panic("should not be reached") -} -func (mfc *monitorFakeCmd) SetEnv(env []string) { - panic("should not be reached") -} -func (mfc *monitorFakeCmd) StdoutPipe() (io.ReadCloser, error) { - panic("should not be reached") -} -func (mfc *monitorFakeCmd) StderrPipe() (io.ReadCloser, error) { - panic("should not be reached") -} -func (mfc *monitorFakeCmd) Start() error { - panic("should not be reached") -} -func (mfc *monitorFakeCmd) Wait() error { - panic("should not be reached") -} -func (mfc *monitorFakeCmd) Stop() { - panic("should not be reached") -} - -func TestIPTablesMonitor(t *testing.T) { - mfe := newMonitorFakeExec() - ipt := New(mfe, ProtocolIpv4) - - var reloads uint32 - stopCh := make(chan struct{}) - - canary := Chain("MONITOR-TEST-CANARY") - tables := []Table{TableMangle, TableFilter, TableNAT} - go ipt.Monitor(canary, tables, func() { - if !ensureNoChains(mfe) { - t.Errorf("reload called while canaries still exist") - } - atomic.AddUint32(&reloads, 1) - }, 100*time.Millisecond, stopCh) - - // Monitor should create canary chains quickly - if err := waitForChains(mfe, canary, tables); err != nil { - t.Errorf("failed to create iptables canaries: %v", err) - } - - if err := waitForReloads(&reloads, 0); err != nil { - t.Errorf("got unexpected reloads: %v", err) - } - - // If we delete all of the chains, it should reload - ipt.DeleteChain(TableMangle, canary) - ipt.DeleteChain(TableFilter, canary) - ipt.DeleteChain(TableNAT, canary) - - if err := waitForReloads(&reloads, 1); err != nil { - t.Errorf("got unexpected number of reloads after flush: %v", err) - } - if err := waitForChains(mfe, canary, tables); err != nil { - t.Errorf("failed to create iptables canaries: %v", err) - } - - // If we delete two chains, it should not reload yet - ipt.DeleteChain(TableMangle, canary) - ipt.DeleteChain(TableFilter, canary) - - if err := waitForNoReload(&reloads, 1); err != nil { - t.Errorf("got unexpected number of reloads after partial flush: %v", err) - } - - // Now ensure that "iptables -L" will get an error about the xtables.lock, and - // delete the last chain. The monitor should not reload, because it can't actually - // tell if the chain was deleted or not. - mfe.blockIPTables(true) - ipt.DeleteChain(TableNAT, canary) - if err := waitForBlocked(mfe); err != nil { - t.Errorf("failed waiting for monitor to be blocked from monitoring: %v", err) - } - - // After unblocking the monitor, it should now reload - mfe.blockIPTables(false) - - if err := waitForReloads(&reloads, 2); err != nil { - t.Errorf("got unexpected number of reloads after slow flush: %v", err) - } - if err := waitForChains(mfe, canary, tables); err != nil { - t.Errorf("failed to create iptables canaries: %v", err) - } - - // If we close the stop channel, it should stop running - close(stopCh) - - if err := waitForNoReload(&reloads, 2); err != nil { - t.Errorf("got unexpected number of reloads after stop: %v", err) - } - if !ensureNoChains(mfe) { - t.Errorf("canaries still exist after stopping monitor") - } - - // If we create a new monitor while the iptables lock is held, it will - // retry creating canaries until it succeeds - - stopCh = make(chan struct{}) - _ = mfe.getWasBlocked() - mfe.blockIPTables(true) - go ipt.Monitor(canary, tables, func() { - if !ensureNoChains(mfe) { - t.Errorf("reload called while canaries still exist") - } - atomic.AddUint32(&reloads, 1) - }, 100*time.Millisecond, stopCh) - - // Monitor should not have created canaries yet - if !ensureNoChains(mfe) { - t.Errorf("canary created while iptables blocked") - } - - if err := waitForBlocked(mfe); err != nil { - t.Errorf("failed waiting for monitor to fail creating canaries: %v", err) - } - - mfe.blockIPTables(false) - if err := waitForChains(mfe, canary, tables); err != nil { - t.Errorf("failed to create iptables canaries: %v", err) - } - - close(stopCh) -} - -func waitForChains(mfe *monitorFakeExec, canary Chain, tables []Table) error { - return utilwait.PollImmediate(100*time.Millisecond, time.Second, func() (bool, error) { - mfe.Lock() - defer mfe.Unlock() - - for _, table := range tables { - if !mfe.tables[string(table)].Has(string(canary)) { - return false, nil - } - } - return true, nil - }) -} - -func ensureNoChains(mfe *monitorFakeExec) bool { - mfe.Lock() - defer mfe.Unlock() - return mfe.tables["mangle"].Len() == 0 && - mfe.tables["filter"].Len() == 0 && - mfe.tables["nat"].Len() == 0 -} - -func waitForReloads(reloads *uint32, expected uint32) error { - if atomic.LoadUint32(reloads) < expected { - utilwait.PollImmediate(100*time.Millisecond, time.Second, func() (bool, error) { - return atomic.LoadUint32(reloads) >= expected, nil - }) - } - got := atomic.LoadUint32(reloads) - if got != expected { - return fmt.Errorf("expected %d, got %d", expected, got) - } - return nil -} - -func waitForNoReload(reloads *uint32, expected uint32) error { - utilwait.PollImmediate(50*time.Millisecond, 250*time.Millisecond, func() (bool, error) { - return atomic.LoadUint32(reloads) > expected, nil - }) - - got := atomic.LoadUint32(reloads) - if got != expected { - return fmt.Errorf("expected %d, got %d", expected, got) - } - return nil -} - -func waitForBlocked(mfe *monitorFakeExec) error { - return utilwait.PollImmediate(100*time.Millisecond, time.Second, func() (bool, error) { - blocked := mfe.getWasBlocked() - return blocked, nil - }) -} diff --git a/pkg/util/iptables/save_restore.go b/pkg/util/iptables/save_restore.go index 7ec11e4f00f..b788beb9113 100644 --- a/pkg/util/iptables/save_restore.go +++ b/pkg/util/iptables/save_restore.go @@ -21,102 +21,32 @@ import ( "fmt" ) -var ( - commitBytes = []byte("COMMIT") - spaceBytes = []byte(" ") -) - // MakeChainLine return an iptables-save/restore formatted chain line given a Chain func MakeChainLine(chain Chain) string { return fmt.Sprintf(":%s - [0:0]", chain) } -// GetChainLines parses a table's iptables-save data to find chains in the table. -// It returns a map of iptables.Chain to []byte where the []byte is the chain line -// from save (with counters etc.). -// Note that to avoid allocations memory is SHARED with save. -func GetChainLines(table Table, save []byte) map[Chain][]byte { - chainsMap := make(map[Chain][]byte) - tablePrefix := []byte("*" + string(table)) - readIndex := 0 - // find beginning of table - for readIndex < len(save) { - line, n := readLine(readIndex, save) - readIndex = n - if bytes.HasPrefix(line, tablePrefix) { +// GetChainsFromTable parses iptables-save data to find the chains that are defined. It +// assumes that save contains a single table's data, and returns a map with keys for every +// chain defined in that table. +func GetChainsFromTable(save []byte) map[Chain]struct{} { + chainsMap := make(map[Chain]struct{}) + + for { + i := bytes.Index(save, []byte("\n:")) + if i == -1 { break } - } - // parse table lines - for readIndex < len(save) { - line, n := readLine(readIndex, save) - readIndex = n - if len(line) == 0 { - continue - } - if bytes.HasPrefix(line, commitBytes) || line[0] == '*' { + start := i + 2 + save = save[start:] + end := bytes.Index(save, []byte(" ")) + if i == -1 { + // shouldn't happen, but... break - } else if line[0] == '#' { - continue - } else if line[0] == ':' && len(line) > 1 { - // We assume that the contains space - chain lines have 3 fields, - // space delimited. If there is no space, this line will panic. - spaceIndex := bytes.Index(line, spaceBytes) - if spaceIndex == -1 { - panic(fmt.Sprintf("Unexpected chain line in iptables-save output: %v", string(line))) - } - chain := Chain(line[1:spaceIndex]) - chainsMap[chain] = line } + chain := Chain(save[:end]) + chainsMap[chain] = struct{}{} + save = save[end:] } return chainsMap } - -func readLine(readIndex int, byteArray []byte) ([]byte, int) { - currentReadIndex := readIndex - - // consume left spaces - for currentReadIndex < len(byteArray) { - if byteArray[currentReadIndex] == ' ' { - currentReadIndex++ - } else { - break - } - } - - // leftTrimIndex stores the left index of the line after the line is left-trimmed - leftTrimIndex := currentReadIndex - - // rightTrimIndex stores the right index of the line after the line is right-trimmed - // it is set to -1 since the correct value has not yet been determined. - rightTrimIndex := -1 - - for ; currentReadIndex < len(byteArray); currentReadIndex++ { - if byteArray[currentReadIndex] == ' ' { - // set rightTrimIndex - if rightTrimIndex == -1 { - rightTrimIndex = currentReadIndex - } - } else if (byteArray[currentReadIndex] == '\n') || (currentReadIndex == (len(byteArray) - 1)) { - // end of line or byte buffer is reached - if currentReadIndex <= leftTrimIndex { - return nil, currentReadIndex + 1 - } - // set the rightTrimIndex - if rightTrimIndex == -1 { - rightTrimIndex = currentReadIndex - if currentReadIndex == (len(byteArray)-1) && (byteArray[currentReadIndex] != '\n') { - // ensure that the last character is part of the returned string, - // unless the last character is '\n' - rightTrimIndex = currentReadIndex + 1 - } - } - // Avoid unnecessary allocation. - return byteArray[leftTrimIndex:rightTrimIndex], currentReadIndex + 1 - } else { - // unset rightTrimIndex - rightTrimIndex = -1 - } - } - return nil, currentReadIndex -} diff --git a/pkg/util/iptables/save_restore_test.go b/pkg/util/iptables/save_restore_test.go index d7c89d3e42f..2a4c383e12f 100644 --- a/pkg/util/iptables/save_restore_test.go +++ b/pkg/util/iptables/save_restore_test.go @@ -18,36 +18,81 @@ package iptables import ( "testing" + + "github.com/lithammer/dedent" ) -func TestReadLinesFromByteBuffer(t *testing.T) { - testFn := func(byteArray []byte, expected []string) { - index := 0 - readIndex := 0 - for ; readIndex < len(byteArray); index++ { - line, n := readLine(readIndex, byteArray) - readIndex = n - if expected[index] != string(line) { - t.Errorf("expected:%q, actual:%q", expected[index], line) - } - } // for - if readIndex < len(byteArray) { - t.Errorf("Byte buffer was only partially read. Buffer length is:%d, readIndex is:%d", len(byteArray), readIndex) +func checkChains(t *testing.T, save []byte, expected map[Chain]struct{}) { + chains := GetChainsFromTable(save) + for chain := range expected { + if _, exists := chains[chain]; !exists { + t.Errorf("GetChainsFromTable expected chain not present: %s", chain) } - if index < len(expected) { - t.Errorf("All expected strings were not compared. expected arr length:%d, matched count:%d", len(expected), index-1) + } + for chain := range chains { + if _, exists := expected[chain]; !exists { + t.Errorf("GetChainsFromTable chain unexpectedly present: %s", chain) } } +} - byteArray1 := []byte("\n Line 1 \n\n\n L ine4 \nLine 5 \n \n") - expected1 := []string{"", "Line 1", "", "", "L ine4", "Line 5", ""} - testFn(byteArray1, expected1) - - byteArray1 = []byte("") - expected1 = []string{} - testFn(byteArray1, expected1) - - byteArray1 = []byte("\n\n") - expected1 = []string{"", ""} - testFn(byteArray1, expected1) +func TestGetChainsFromTable(t *testing.T) { + iptablesSave := dedent.Dedent(` + # Generated by iptables-save v1.4.21 on Fri Aug 7 14:47:37 2015 + *nat + :PREROUTING ACCEPT [2:138] + :INPUT ACCEPT [0:0] + :OUTPUT ACCEPT [0:0] + :POSTROUTING ACCEPT [0:0] + :DOCKER - [0:0] + :KUBE-NODEPORT-CONTAINER - [0:0] + :KUBE-NODEPORT-HOST - [0:0] + :KUBE-PORTALS-CONTAINER - [0:0] + :KUBE-PORTALS-HOST - [0:0] + :KUBE-SVC-1111111111111111 - [0:0] + :KUBE-SVC-2222222222222222 - [0:0] + :KUBE-SVC-3333333333333333 - [0:0] + :KUBE-SVC-4444444444444444 - [0:0] + :KUBE-SVC-5555555555555555 - [0:0] + :KUBE-SVC-6666666666666666 - [0:0] + -A PREROUTING -m comment --comment "handle ClusterIPs; NOTE: this must be before the NodePort rules" -j KUBE-PORTALS-CONTAINER + -A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER + -A PREROUTING -m addrtype --dst-type LOCAL -m comment --comment "handle service NodePorts; NOTE: this must be the last rule in the chain" -j KUBE-NODEPORT-CONTAINER + -A OUTPUT -m comment --comment "handle ClusterIPs; NOTE: this must be before the NodePort rules" -j KUBE-PORTALS-HOST + -A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER + -A OUTPUT -m addrtype --dst-type LOCAL -m comment --comment "handle service NodePorts; NOTE: this must be the last rule in the chain" -j KUBE-NODEPORT-HOST + -A POSTROUTING -s 10.246.1.0/24 ! -o cbr0 -j MASQUERADE + -A POSTROUTING -s 10.0.2.15 -d 10.0.2.15 -m comment --comment "handle pod connecting to self" -j MASQUERADE + -A KUBE-PORTALS-CONTAINER -d 10.247.0.1 -p tcp -m comment --comment "portal for default/kubernetes:" -m state --state NEW -m tcp --dport 443 -j KUBE-SVC-5555555555555555 + -A KUBE-PORTALS-CONTAINER -d 10.247.0.10 -p udp -m comment --comment "portal for kube-system/kube-dns:dns" -m state --state NEW -m udp --dport 53 -j KUBE-SVC-6666666666666666 + -A KUBE-PORTALS-CONTAINER -d 10.247.0.10 -p tcp -m comment --comment "portal for kube-system/kube-dns:dns-tcp" -m state --state NEW -m tcp --dport 53 -j KUBE-SVC-2222222222222222 + -A KUBE-PORTALS-HOST -d 10.247.0.1 -p tcp -m comment --comment "portal for default/kubernetes:" -m state --state NEW -m tcp --dport 443 -j KUBE-SVC-5555555555555555 + -A KUBE-PORTALS-HOST -d 10.247.0.10 -p udp -m comment --comment "portal for kube-system/kube-dns:dns" -m state --state NEW -m udp --dport 53 -j KUBE-SVC-6666666666666666 + -A KUBE-PORTALS-HOST -d 10.247.0.10 -p tcp -m comment --comment "portal for kube-system/kube-dns:dns-tcp" -m state --state NEW -m tcp --dport 53 -j KUBE-SVC-2222222222222222 + -A KUBE-SVC-1111111111111111 -p udp -m comment --comment "kube-system/kube-dns:dns" -m recent --set --name KUBE-SVC-1111111111111111 --mask 255.255.255.255 --rsource -j DNAT --to-destination 10.246.1.2:53 + -A KUBE-SVC-2222222222222222 -m comment --comment "kube-system/kube-dns:dns-tcp" -j KUBE-SVC-3333333333333333 + -A KUBE-SVC-3333333333333333 -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp" -m recent --set --name KUBE-SVC-3333333333333333 --mask 255.255.255.255 --rsource -j DNAT --to-destination 10.246.1.2:53 + -A KUBE-SVC-4444444444444444 -p tcp -m comment --comment "default/kubernetes:" -m recent --set --name KUBE-SVC-4444444444444444 --mask 255.255.255.255 --rsource -j DNAT --to-destination 10.245.1.2:443 + -A KUBE-SVC-5555555555555555 -m comment --comment "default/kubernetes:" -j KUBE-SVC-4444444444444444 + -A KUBE-SVC-6666666666666666 -m comment --comment "kube-system/kube-dns:dns" -j KUBE-SVC-1111111111111111 + COMMIT + `) + expected := map[Chain]struct{}{ + ChainPrerouting: {}, + Chain("INPUT"): {}, + Chain("OUTPUT"): {}, + ChainPostrouting: {}, + Chain("DOCKER"): {}, + Chain("KUBE-NODEPORT-CONTAINER"): {}, + Chain("KUBE-NODEPORT-HOST"): {}, + Chain("KUBE-PORTALS-CONTAINER"): {}, + Chain("KUBE-PORTALS-HOST"): {}, + Chain("KUBE-SVC-1111111111111111"): {}, + Chain("KUBE-SVC-2222222222222222"): {}, + Chain("KUBE-SVC-3333333333333333"): {}, + Chain("KUBE-SVC-4444444444444444"): {}, + Chain("KUBE-SVC-5555555555555555"): {}, + Chain("KUBE-SVC-6666666666666666"): {}, + } + checkChains(t, []byte(iptablesSave), expected) } diff --git a/pkg/util/iptables/testing/fake.go b/pkg/util/iptables/testing/fake.go index 20e216186f1..51b1048cdc5 100644 --- a/pkg/util/iptables/testing/fake.go +++ b/pkg/util/iptables/testing/fake.go @@ -22,111 +22,297 @@ import ( "strings" "time" - "github.com/openyurtio/openyurt/pkg/util/iptables" -) + "k8s.io/apimachinery/pkg/util/sets" -const ( - // Destination represents the destination address flag - Destination = "-d " - // Source represents the source address flag - Source = "-s " - // DPort represents the destination port flag - DPort = "--dport " - // Protocol represents the protocol flag - Protocol = "-p " - // Jump represents jump flag specifies the jump target - Jump = "-j " - // Reject specifies the reject target - Reject = "REJECT" - // ToDest represents the flag used to specify the destination address in DNAT - ToDest = "--to-destination " - // Recent represents the sub-command recent that allows to dynamically create list of IP address to match against - Recent = "recent " - // MatchSet represents the flag which match packets against the specified set - MatchSet = "--match-set " - // SrcType represents the --src-type flag which matches if the source address is of given type - SrcType = "--src-type " - // Masquerade represents the target that is used in nat table. - Masquerade = "MASQUERADE " + "github.com/openyurtio/openyurt/pkg/util/iptables" ) -// Rule holds a map of rules. -type Rule map[string]string - // FakeIPTables is no-op implementation of iptables Interface. type FakeIPTables struct { hasRandomFully bool - Lines []byte - ipv6 bool + protocol iptables.Protocol + + Dump *IPTablesDump } // NewFake returns a no-op iptables.Interface func NewFake() *FakeIPTables { - return &FakeIPTables{} + f := &FakeIPTables{ + protocol: iptables.ProtocolIPv4, + Dump: &IPTablesDump{ + Tables: []Table{ + { + Name: iptables.TableNAT, + Chains: []Chain{ + {Name: iptables.ChainPrerouting}, + {Name: iptables.ChainInput}, + {Name: iptables.ChainOutput}, + {Name: iptables.ChainPostrouting}, + }, + }, + { + Name: iptables.TableFilter, + Chains: []Chain{ + {Name: iptables.ChainInput}, + {Name: iptables.ChainForward}, + {Name: iptables.ChainOutput}, + }, + }, + { + Name: iptables.TableMangle, + Chains: []Chain{}, + }, + }, + }, + } + + return f } -// NewIpv6Fake returns a no-op iptables.Interface with IsIPv6() == true -func NewIpv6Fake() *FakeIPTables { - return &FakeIPTables{ipv6: true} +// NewIPv6Fake returns a no-op iptables.Interface with IsIPv6() == true +func NewIPv6Fake() *FakeIPTables { + f := NewFake() + f.protocol = iptables.ProtocolIPv6 + return f } -// SetHasRandomFully is part of iptables.Interface +// SetHasRandomFully sets f's return value for HasRandomFully() func (f *FakeIPTables) SetHasRandomFully(can bool) *FakeIPTables { f.hasRandomFully = can return f } // EnsureChain is part of iptables.Interface -func (*FakeIPTables) EnsureChain(table iptables.Table, chain iptables.Chain) (bool, error) { - return true, nil +func (f *FakeIPTables) EnsureChain(table iptables.Table, chain iptables.Chain) (bool, error) { + t, err := f.Dump.GetTable(table) + if err != nil { + return false, err + } + if c, _ := f.Dump.GetChain(table, chain); c != nil { + return true, nil + } + t.Chains = append(t.Chains, Chain{Name: chain}) + return false, nil } // FlushChain is part of iptables.Interface -func (*FakeIPTables) FlushChain(table iptables.Table, chain iptables.Chain) error { +func (f *FakeIPTables) FlushChain(table iptables.Table, chain iptables.Chain) error { + if c, _ := f.Dump.GetChain(table, chain); c != nil { + c.Rules = nil + } return nil } // DeleteChain is part of iptables.Interface -func (*FakeIPTables) DeleteChain(table iptables.Table, chain iptables.Chain) error { +func (f *FakeIPTables) DeleteChain(table iptables.Table, chain iptables.Chain) error { + t, err := f.Dump.GetTable(table) + if err != nil { + return err + } + for i := range t.Chains { + if t.Chains[i].Name == chain { + t.Chains = append(t.Chains[:i], t.Chains[i+1:]...) + return nil + } + } return nil } +// ChainExists is part of iptables.Interface +func (f *FakeIPTables) ChainExists(table iptables.Table, chain iptables.Chain) (bool, error) { + if _, err := f.Dump.GetTable(table); err != nil { + return false, err + } + if c, _ := f.Dump.GetChain(table, chain); c != nil { + return true, nil + } + return false, nil +} + // EnsureRule is part of iptables.Interface -func (*FakeIPTables) EnsureRule(position iptables.RulePosition, table iptables.Table, chain iptables.Chain, args ...string) (bool, error) { - return true, nil +func (f *FakeIPTables) EnsureRule(position iptables.RulePosition, table iptables.Table, chain iptables.Chain, args ...string) (bool, error) { + c, err := f.Dump.GetChain(table, chain) + if err != nil { + return false, err + } + + rule := "-A " + string(chain) + " " + strings.Join(args, " ") + for _, r := range c.Rules { + if r.Raw == rule { + return true, nil + } + } + + parsed, err := ParseRule(rule, false) + if err != nil { + return false, err + } + + if position == iptables.Append { + c.Rules = append(c.Rules, parsed) + } else { + c.Rules = append([]*Rule{parsed}, c.Rules...) + } + return false, nil } // DeleteRule is part of iptables.Interface -func (*FakeIPTables) DeleteRule(table iptables.Table, chain iptables.Chain, args ...string) error { +func (f *FakeIPTables) DeleteRule(table iptables.Table, chain iptables.Chain, args ...string) error { + c, err := f.Dump.GetChain(table, chain) + if err != nil { + return err + } + + rule := "-A " + string(chain) + " " + strings.Join(args, " ") + for i, r := range c.Rules { + if r.Raw == rule { + c.Rules = append(c.Rules[:i], c.Rules[i+1:]...) + break + } + } return nil } -// IsIpv6 is part of iptables.Interface -func (f *FakeIPTables) IsIpv6() bool { - return f.ipv6 +// IsIPv6 is part of iptables.Interface +func (f *FakeIPTables) IsIPv6() bool { + return f.protocol == iptables.ProtocolIPv6 } -// Save is part of iptables.Interface -func (f *FakeIPTables) Save(table iptables.Table) ([]byte, error) { - lines := make([]byte, len(f.Lines)) - copy(lines, f.Lines) - return lines, nil +// Protocol is part of iptables.Interface +func (f *FakeIPTables) Protocol() iptables.Protocol { + return f.protocol +} + +func (f *FakeIPTables) saveTable(table iptables.Table, buffer *bytes.Buffer) error { + t, err := f.Dump.GetTable(table) + if err != nil { + return err + } + + fmt.Fprintf(buffer, "*%s\n", table) + for _, c := range t.Chains { + fmt.Fprintf(buffer, ":%s - [%d:%d]\n", c.Name, c.Packets, c.Bytes) + } + for _, c := range t.Chains { + for _, r := range c.Rules { + fmt.Fprintf(buffer, "%s\n", r.Raw) + } + } + fmt.Fprintf(buffer, "COMMIT\n") + return nil } // SaveInto is part of iptables.Interface func (f *FakeIPTables) SaveInto(table iptables.Table, buffer *bytes.Buffer) error { - buffer.Write(f.Lines) + if table == "" { + // As a secret extension to the API, FakeIPTables treats table="" as + // meaning "all tables" + for i := range f.Dump.Tables { + err := f.saveTable(f.Dump.Tables[i].Name, buffer) + if err != nil { + return err + } + } + return nil + } + + return f.saveTable(table, buffer) +} + +// This is not a complete list but it's enough to pass the unit tests +var builtinTargets = sets.NewString("ACCEPT", "DROP", "RETURN", "REJECT", "DNAT", "SNAT", "MASQUERADE", "MARK") + +func (f *FakeIPTables) restoreTable(newDump *IPTablesDump, newTable *Table, flush iptables.FlushFlag, counters iptables.RestoreCountersFlag) error { + oldTable, err := f.Dump.GetTable(newTable.Name) + if err != nil { + return err + } + + backupChains := make([]Chain, len(oldTable.Chains)) + copy(backupChains, oldTable.Chains) + + // Update internal state + if flush == iptables.FlushTables { + oldTable.Chains = make([]Chain, 0, len(newTable.Chains)) + } + for _, newChain := range newTable.Chains { + oldChain, _ := f.Dump.GetChain(newTable.Name, newChain.Name) + switch { + case oldChain == nil && newChain.Deleted: + // no-op + case oldChain == nil && !newChain.Deleted: + oldTable.Chains = append(oldTable.Chains, newChain) + case oldChain != nil && newChain.Deleted: + _ = f.DeleteChain(newTable.Name, newChain.Name) + case oldChain != nil && !newChain.Deleted: + // replace old data with new + oldChain.Rules = newChain.Rules + if counters == iptables.RestoreCounters { + oldChain.Packets = newChain.Packets + oldChain.Bytes = newChain.Bytes + } + } + } + + // Now check that all old/new jumps are valid + for _, chain := range oldTable.Chains { + for _, rule := range chain.Rules { + if rule.Jump == nil { + continue + } + if builtinTargets.Has(rule.Jump.Value) { + continue + } + + jumpedChain, _ := f.Dump.GetChain(oldTable.Name, iptables.Chain(rule.Jump.Value)) + if jumpedChain == nil { + newChain, _ := newDump.GetChain(oldTable.Name, iptables.Chain(rule.Jump.Value)) + if newChain != nil { + // rule is an old rule that jumped to a chain which + // was deleted by newDump. + oldTable.Chains = backupChains + return fmt.Errorf("deleted chain %q is referenced by existing rules", newChain.Name) + } else { + // rule is a new rule that jumped to a chain that was + // neither created nor pre-existing + oldTable.Chains = backupChains + return fmt.Errorf("rule %q jumps to a non-existent chain", rule.Raw) + } + } + } + } + return nil } // Restore is part of iptables.Interface -func (*FakeIPTables) Restore(table iptables.Table, data []byte, flush iptables.FlushFlag, counters iptables.RestoreCountersFlag) error { - return nil +func (f *FakeIPTables) Restore(table iptables.Table, data []byte, flush iptables.FlushFlag, counters iptables.RestoreCountersFlag) error { + dump, err := ParseIPTablesDump(string(data)) + if err != nil { + return err + } + + newTable, err := dump.GetTable(table) + if err != nil { + return err + } + + return f.restoreTable(dump, newTable, flush, counters) } // RestoreAll is part of iptables.Interface func (f *FakeIPTables) RestoreAll(data []byte, flush iptables.FlushFlag, counters iptables.RestoreCountersFlag) error { - f.Lines = data + dump, err := ParseIPTablesDump(string(data)) + if err != nil { + return err + } + + for i := range dump.Tables { + err = f.restoreTable(dump, &dump.Tables[i], flush, counters) + if err != nil { + return err + } + } return nil } @@ -134,34 +320,13 @@ func (f *FakeIPTables) RestoreAll(data []byte, flush iptables.FlushFlag, counter func (f *FakeIPTables) Monitor(canary iptables.Chain, tables []iptables.Table, reloadFunc func(), interval time.Duration, stopCh <-chan struct{}) { } -func getToken(line, separator string) string { - tokens := strings.Split(line, separator) - if len(tokens) == 2 { - return strings.Split(tokens[1], " ")[0] - } - return "" -} - -// GetRules is part of iptables.Interface -func (f *FakeIPTables) GetRules(chainName string) (rules []Rule) { - for _, l := range strings.Split(string(f.Lines), "\n") { - if strings.Contains(l, fmt.Sprintf("-A %v", chainName)) { - newRule := Rule(map[string]string{}) - for _, arg := range []string{Destination, Source, DPort, Protocol, Jump, ToDest, Recent, MatchSet, SrcType, Masquerade} { - tok := getToken(l, arg) - if tok != "" { - newRule[arg] = tok - } - } - rules = append(rules, newRule) - } - } - return -} - // HasRandomFully is part of iptables.Interface func (f *FakeIPTables) HasRandomFully() bool { return f.hasRandomFully } +func (f *FakeIPTables) Present() bool { + return true +} + var _ = iptables.Interface(&FakeIPTables{}) diff --git a/pkg/util/iptables/testing/fake_test.go b/pkg/util/iptables/testing/fake_test.go index 93924095570..d139bb9b433 100644 --- a/pkg/util/iptables/testing/fake_test.go +++ b/pkg/util/iptables/testing/fake_test.go @@ -1,5 +1,5 @@ /* -Copyright 2015 The Kubernetes Authors. +Copyright 2022 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,598 +18,328 @@ package testing import ( "bytes" - "reflect" + "strings" "testing" - "time" - "github.com/openyurtio/openyurt/pkg/util/iptables" -) + "github.com/lithammer/dedent" -const ( - failed = "\u2717" - succeed = "\u2713" + "github.com/openyurtio/openyurt/pkg/util/iptables" ) -func TestNewFake(t *testing.T) { - tests := []struct { - name string - expect *FakeIPTables - }{ - { - "normal", - &FakeIPTables{}, - }, - } - - for _, tt := range tests { - tf := func(t *testing.T) { - t.Parallel() - t.Logf("\tTestCase: %s", tt.name) - { - get := NewFake() - - if !reflect.DeepEqual(*tt.expect, *get) { - t.Fatalf("\t%s\texpect %v, but get %v", failed, tt.expect, get) - } - t.Logf("\t%s\texpect %v, get %v", succeed, tt.expect, get) - - } - } - t.Run(tt.name, tf) - } -} - -func TestNewIpv6Fake(t *testing.T) { - tests := []struct { - name string - expect *FakeIPTables - }{ - { - "normal", - &FakeIPTables{ipv6: true}, - }, - } - - for _, tt := range tests { - tf := func(t *testing.T) { - t.Parallel() - t.Logf("\tTestCase: %s", tt.name) - { - get := NewIpv6Fake() - - if !reflect.DeepEqual(*tt.expect, *get) { - t.Fatalf("\t%s\texpect %v, but get %v", failed, tt.expect, get) - } - t.Logf("\t%s\texpect %v, get %v", succeed, tt.expect, get) - - } - } - t.Run(tt.name, tf) - } -} - -func TestSetHasRandomFully(t *testing.T) { - f := FakeIPTables{} - - tests := []struct { - name string - can bool - expect *FakeIPTables - }{ - { - "normal", - true, - &FakeIPTables{ - hasRandomFully: true, - ipv6: false, - }, - }, - } - - for _, tt := range tests { - tf := func(t *testing.T) { - t.Parallel() - t.Logf("\tTestCase: %s", tt.name) - { - var get = f.SetHasRandomFully(true) - if !reflect.DeepEqual(*tt.expect, *get) { - t.Fatalf("\t%s\texpect %v, but get %v", failed, tt.expect, get) - } - t.Logf("\t%s\texpect %v, get %v", succeed, tt.expect, get) - - } - } - t.Run(tt.name, tf) - } -} - -func TestEnsureChain(t *testing.T) { - f := FakeIPTables{} - - tests := []struct { - name string - table iptables.Table - chain iptables.Chain - expect bool - }{ - { - "normal", - "", - "", - true, - }, - } - - for _, tt := range tests { - tf := func(t *testing.T) { - t.Parallel() - t.Logf("\tTestCase: %s", tt.name) - { - get, _ := f.EnsureChain(tt.table, tt.chain) - if !reflect.DeepEqual(tt.expect, get) { - t.Fatalf("\t%s\texpect %v, but get %v", failed, tt.expect, get) - } - t.Logf("\t%s\texpect %v, get %v", succeed, tt.expect, get) - - } - } - t.Run(tt.name, tf) - } -} - -func TestFlushChain(t *testing.T) { - f := FakeIPTables{} - - tests := []struct { - name string - table iptables.Table - chain iptables.Chain - expect error - }{ - { - "normal", - "", - "", - nil, - }, - } - - for _, tt := range tests { - tf := func(t *testing.T) { - t.Parallel() - t.Logf("\tTestCase: %s", tt.name) - { - get := f.FlushChain(tt.table, tt.chain) - if !reflect.DeepEqual(tt.expect, get) { - t.Fatalf("\t%s\texpect %v, but get %v", failed, tt.expect, get) - } - t.Logf("\t%s\texpect %v, get %v", succeed, tt.expect, get) - - } - } - t.Run(tt.name, tf) - } -} - -func TestDeleteChain(t *testing.T) { - f := FakeIPTables{} - - tests := []struct { - name string - table iptables.Table - chain iptables.Chain - expect error - }{ - { - "normal", - "", - "", - nil, - }, - } - - for _, tt := range tests { - tf := func(t *testing.T) { - t.Parallel() - t.Logf("\tTestCase: %s", tt.name) - { - get := f.DeleteChain(tt.table, tt.chain) - if !reflect.DeepEqual(tt.expect, get) { - t.Fatalf("\t%s\texpect %v, but get %v", failed, tt.expect, get) - } - t.Logf("\t%s\texpect %v, get %v", succeed, tt.expect, get) - - } - } - t.Run(tt.name, tf) - } -} - -func TestEnsureRule(t *testing.T) { - f := FakeIPTables{} - - tests := []struct { - name string - position iptables.RulePosition - table iptables.Table - chain iptables.Chain - expect error - }{ - { - "normal", - "", - "", - "", - nil, - }, - } - - for _, tt := range tests { - tf := func(t *testing.T) { - t.Parallel() - t.Logf("\tTestCase: %s", tt.name) - { - _, get := f.EnsureRule(tt.position, tt.table, tt.chain) - if !reflect.DeepEqual(tt.expect, get) { - t.Fatalf("\t%s\texpect %v, but get %v", failed, tt.expect, get) - } - t.Logf("\t%s\texpect %v, get %v", succeed, tt.expect, get) - - } - } - t.Run(tt.name, tf) - } -} - -func TestDeleteRule(t *testing.T) { - f := FakeIPTables{} - - tests := []struct { - name string - table iptables.Table - chain iptables.Chain - expect error - }{ - { - "normal", - "", - "", - nil, - }, - } - - for _, tt := range tests { - tf := func(t *testing.T) { - t.Parallel() - t.Logf("\tTestCase: %s", tt.name) - { - get := f.DeleteRule(tt.table, tt.chain) - if !reflect.DeepEqual(tt.expect, get) { - t.Fatalf("\t%s\texpect %v, but get %v", failed, tt.expect, get) - } - t.Logf("\t%s\texpect %v, get %v", succeed, tt.expect, get) - - } - } - t.Run(tt.name, tf) - } -} - -func TestIsIpv6(t *testing.T) { - f := FakeIPTables{ - ipv6: true, - } - - tests := []struct { - name string - expect bool - }{ - { - "normal", - true, - }, - } - - for _, tt := range tests { - tf := func(t *testing.T) { - t.Parallel() - t.Logf("\tTestCase: %s", tt.name) - { - get := f.IsIpv6() - if !reflect.DeepEqual(tt.expect, get) { - t.Fatalf("\t%s\texpect %v, but get %v", failed, tt.expect, get) - } - t.Logf("\t%s\texpect %v, get %v", succeed, tt.expect, get) - - } - } - t.Run(tt.name, tf) - } -} - -func TestSave(t *testing.T) { - f := FakeIPTables{} - - tests := []struct { - name string - table iptables.Table - expect error - }{ - { - "normal", - "", - nil, - }, - } - - for _, tt := range tests { - tf := func(t *testing.T) { - t.Parallel() - t.Logf("\tTestCase: %s", tt.name) - { - _, get := f.Save(tt.table) - if !reflect.DeepEqual(tt.expect, get) { - t.Fatalf("\t%s\texpect %v, but get %v", failed, tt.expect, get) - } - t.Logf("\t%s\texpect %v, get %v", succeed, tt.expect, get) - - } - } - t.Run(tt.name, tf) - } -} - -func TestSaveInto(t *testing.T) { - f := FakeIPTables{} - - tests := []struct { - name string - table iptables.Table - buffer *bytes.Buffer - expect error - }{ - { - "normal", - "", - &bytes.Buffer{}, - nil, - }, - } - - for _, tt := range tests { - tf := func(t *testing.T) { - t.Parallel() - t.Logf("\tTestCase: %s", tt.name) - { - get := f.SaveInto(tt.table, tt.buffer) - if !reflect.DeepEqual(tt.expect, get) { - t.Fatalf("\t%s\texpect %v, but get %v", failed, tt.expect, get) - } - t.Logf("\t%s\texpect %v, get %v", succeed, tt.expect, get) - - } - } - t.Run(tt.name, tf) - } -} - -func TestRestore(t *testing.T) { - f := FakeIPTables{} - - tests := []struct { - name string - table iptables.Table - data []byte - flush iptables.FlushFlag - counters iptables.RestoreCountersFlag - expect error - }{ - { - "normal", - "", - []byte{}, - true, - true, - nil, - }, - } - - for _, tt := range tests { - tf := func(t *testing.T) { - t.Parallel() - t.Logf("\tTestCase: %s", tt.name) - { - get := f.Restore(tt.table, tt.data, tt.flush, tt.counters) - if !reflect.DeepEqual(tt.expect, get) { - t.Fatalf("\t%s\texpect %v, but get %v", failed, tt.expect, get) - } - t.Logf("\t%s\texpect %v, get %v", succeed, tt.expect, get) - - } - } - t.Run(tt.name, tf) - } -} - -func TestRestoreAll(t *testing.T) { - f := FakeIPTables{} - - tests := []struct { - name string - data []byte - flush iptables.FlushFlag - counters iptables.RestoreCountersFlag - expect error - }{ - { - "normal", - []byte{}, - true, - true, - nil, - }, - } - - for _, tt := range tests { - tf := func(t *testing.T) { - t.Parallel() - t.Logf("\tTestCase: %s", tt.name) - { - get := f.RestoreAll(tt.data, tt.flush, tt.counters) - if !reflect.DeepEqual(tt.expect, get) { - t.Fatalf("\t%s\texpect %v, but get %v", failed, tt.expect, get) - } - t.Logf("\t%s\texpect %v, get %v", succeed, tt.expect, get) - - } - } - t.Run(tt.name, tf) - } -} - -func TestMonitor(t *testing.T) { - f := FakeIPTables{} - - tests := []struct { - name string - canary iptables.Chain - tables []iptables.Table - reloadFunc func() - interval time.Duration - stopCh <-chan struct{} - expect error - }{ - { - "normal", - "", - []iptables.Table{}, - func() {}, - 1, - make(<-chan struct{}), - nil, - }, - } - - for _, tt := range tests { - tf := func(t *testing.T) { - t.Parallel() - t.Logf("\tTestCase: %s", tt.name) - { - f.Monitor(tt.canary, tt.tables, tt.reloadFunc, tt.interval, tt.stopCh) - } - } - t.Run(tt.name, tf) - } -} - -func TestGetToken(t *testing.T) { - tests := []struct { - name string - line string - separator string - expect string - }{ - { - "normal", - "a,b", - ",", - "a", - }, - { - "empty", - "", - "", - "", - }, - } - - for _, tt := range tests { - tf := func(t *testing.T) { - t.Parallel() - t.Logf("\tTestCase: %s", tt.name) - { - get := getToken(tt.line, tt.separator) - if !reflect.DeepEqual(tt.expect, get) { - t.Fatalf("\t%s\texpect %v, but get %v", failed, tt.expect, get) - } - t.Logf("\t%s\texpect %v, get %v", succeed, tt.expect, get) - - } - } - t.Run(tt.name, tf) - } -} - -func TestGetRules(t *testing.T) { - f := FakeIPTables{ - Lines: []byte("iptables -A INPUT -p tcp --dport 22 -j ACCEPT"), - } - - tests := []struct { - name string - chainName string - expect []Rule - }{ - { - "normal", - "INPUT", - []Rule{ - map[string]string{ - "--dport ": "22", - "-j ": "ACCEPT", - "-p ": "tcp", - }, - }, - }, - } - - for _, tt := range tests { - tf := func(t *testing.T) { - t.Parallel() - t.Logf("\tTestCase: %s", tt.name) - { - get := f.GetRules("") - t.Log(get) - if !reflect.DeepEqual(tt.expect, get) { - t.Fatalf("\t%s\texpect %v, but get %v", failed, tt.expect, get) - } - t.Logf("\t%s\texpect %v, get %v", succeed, tt.expect, get) - - } - } - t.Run(tt.name, tf) - } -} - -func TestHasRandomFully(t *testing.T) { - f := FakeIPTables{ - hasRandomFully: true, - } - - tests := []struct { - name string - expect bool - }{ - { - "normal", - true, - }, - } - - for _, tt := range tests { - tf := func(t *testing.T) { - t.Parallel() - t.Logf("\tTestCase: %s", tt.name) - { - get := f.HasRandomFully() - t.Log(get) - if !reflect.DeepEqual(tt.expect, get) { - t.Fatalf("\t%s\texpect %v, but get %v", failed, tt.expect, get) - } - t.Logf("\t%s\texpect %v, get %v", succeed, tt.expect, get) - - } - } - t.Run(tt.name, tf) +func TestFakeIPTables(t *testing.T) { + fake := NewFake() + buf := bytes.NewBuffer(nil) + + err := fake.SaveInto("", buf) + if err != nil { + t.Fatalf("unexpected error from SaveInto: %v", err) + } + expected := dedent.Dedent(strings.Trim(` + *nat + :PREROUTING - [0:0] + :INPUT - [0:0] + :OUTPUT - [0:0] + :POSTROUTING - [0:0] + COMMIT + *filter + :INPUT - [0:0] + :FORWARD - [0:0] + :OUTPUT - [0:0] + COMMIT + *mangle + COMMIT + `, "\n")) + if buf.String() != expected { + t.Fatalf("bad initial dump. expected:\n%s\n\ngot:\n%s\n", expected, buf.Bytes()) + } + + // EnsureChain + existed, err := fake.EnsureChain(iptables.Table("blah"), iptables.Chain("KUBE-TEST")) + if err == nil { + t.Errorf("did not get expected error creating chain in non-existent table") + } else if existed { + t.Errorf("wrong return value from EnsureChain with non-existent table") + } + existed, err = fake.EnsureChain(iptables.TableNAT, iptables.Chain("KUBE-TEST")) + if err != nil { + t.Errorf("unexpected error creating chain: %v", err) + } else if existed { + t.Errorf("wrong return value from EnsureChain with non-existent chain") + } + existed, err = fake.EnsureChain(iptables.TableNAT, iptables.Chain("KUBE-TEST")) + if err != nil { + t.Errorf("unexpected error creating chain: %v", err) + } else if !existed { + t.Errorf("wrong return value from EnsureChain with existing chain") + } + + // ChainExists + exists, err := fake.ChainExists(iptables.TableNAT, iptables.Chain("KUBE-TEST")) + if err != nil { + t.Errorf("unexpected error checking chain: %v", err) + } else if !exists { + t.Errorf("wrong return value from ChainExists with existing chain") + } + exists, err = fake.ChainExists(iptables.TableNAT, iptables.Chain("KUBE-TEST-NOT")) + if err != nil { + t.Errorf("unexpected error checking chain: %v", err) + } else if exists { + t.Errorf("wrong return value from ChainExists with non-existent chain") + } + + // EnsureRule + existed, err = fake.EnsureRule(iptables.Append, iptables.Table("blah"), iptables.Chain("KUBE-TEST"), "-j", "ACCEPT") + if err == nil { + t.Errorf("did not get expected error creating rule in non-existent table") + } else if existed { + t.Errorf("wrong return value from EnsureRule with non-existent table") + } + existed, err = fake.EnsureRule(iptables.Append, iptables.TableNAT, iptables.Chain("KUBE-TEST-NOT"), "-j", "ACCEPT") + if err == nil { + t.Errorf("did not get expected error creating rule in non-existent chain") + } else if existed { + t.Errorf("wrong return value from EnsureRule with non-existent chain") + } + existed, err = fake.EnsureRule(iptables.Append, iptables.TableNAT, iptables.Chain("KUBE-TEST"), "-j", "ACCEPT") + if err != nil { + t.Errorf("unexpected error creating rule: %v", err) + } else if existed { + t.Errorf("wrong return value from EnsureRule with non-existent rule") + } + existed, err = fake.EnsureRule(iptables.Prepend, iptables.TableNAT, iptables.Chain("KUBE-TEST"), "-j", "DROP") + if err != nil { + t.Errorf("unexpected error creating rule: %v", err) + } else if existed { + t.Errorf("wrong return value from EnsureRule with non-existent rule") + } + existed, err = fake.EnsureRule(iptables.Append, iptables.TableNAT, iptables.Chain("KUBE-TEST"), "-j", "DROP") + if err != nil { + t.Errorf("unexpected error creating rule: %v", err) + } else if !existed { + t.Errorf("wrong return value from EnsureRule with already-existing rule") + } + + // Sanity-check... + buf.Reset() + err = fake.SaveInto("", buf) + if err != nil { + t.Fatalf("unexpected error from SaveInto: %v", err) + } + expected = dedent.Dedent(strings.Trim(` + *nat + :PREROUTING - [0:0] + :INPUT - [0:0] + :OUTPUT - [0:0] + :POSTROUTING - [0:0] + :KUBE-TEST - [0:0] + -A KUBE-TEST -j DROP + -A KUBE-TEST -j ACCEPT + COMMIT + *filter + :INPUT - [0:0] + :FORWARD - [0:0] + :OUTPUT - [0:0] + COMMIT + *mangle + COMMIT + `, "\n")) + if buf.String() != expected { + t.Fatalf("bad sanity-check dump. expected:\n%s\n\ngot:\n%s\n", expected, buf.Bytes()) + } + + // DeleteRule + err = fake.DeleteRule(iptables.Table("blah"), iptables.Chain("KUBE-TEST"), "-j", "DROP") + if err == nil { + t.Errorf("did not get expected error deleting rule in non-existent table") + } + err = fake.DeleteRule(iptables.TableNAT, iptables.Chain("KUBE-TEST-NOT"), "-j", "DROP") + if err == nil { + t.Errorf("did not get expected error deleting rule in non-existent chain") + } + err = fake.DeleteRule(iptables.TableNAT, iptables.Chain("KUBE-TEST"), "-j", "DROPLET") + if err != nil { + t.Errorf("unexpected error deleting non-existent rule: %v", err) + } + err = fake.DeleteRule(iptables.TableNAT, iptables.Chain("KUBE-TEST"), "-j", "DROP") + if err != nil { + t.Errorf("unexpected error deleting rule: %v", err) + } + + // Restore + rules := dedent.Dedent(strings.Trim(` + *nat + :KUBE-RESTORED - [0:0] + :KUBE-MISC-CHAIN - [0:0] + :KUBE-MISC-TWO - [0:0] + :KUBE-EMPTY - [0:0] + -A KUBE-RESTORED -m comment --comment "restored chain" -j ACCEPT + -A KUBE-MISC-CHAIN -s 1.2.3.4 -j KUBE-MISC-TWO + -A KUBE-MISC-CHAIN -d 5.6.7.8 -j MASQUERADE + -A KUBE-MISC-TWO -j ACCEPT + COMMIT + `, "\n")) + err = fake.Restore(iptables.TableNAT, []byte(rules), iptables.NoFlushTables, iptables.NoRestoreCounters) + if err != nil { + t.Fatalf("unexpected error from Restore: %v", err) + } + + // We used NoFlushTables, so this should leave KUBE-TEST unchanged + buf.Reset() + err = fake.SaveInto("", buf) + if err != nil { + t.Fatalf("unexpected error from SaveInto: %v", err) + } + expected = dedent.Dedent(strings.Trim(` + *nat + :PREROUTING - [0:0] + :INPUT - [0:0] + :OUTPUT - [0:0] + :POSTROUTING - [0:0] + :KUBE-TEST - [0:0] + :KUBE-RESTORED - [0:0] + :KUBE-MISC-CHAIN - [0:0] + :KUBE-MISC-TWO - [0:0] + :KUBE-EMPTY - [0:0] + -A KUBE-TEST -j ACCEPT + -A KUBE-RESTORED -m comment --comment "restored chain" -j ACCEPT + -A KUBE-MISC-CHAIN -s 1.2.3.4 -j KUBE-MISC-TWO + -A KUBE-MISC-CHAIN -d 5.6.7.8 -j MASQUERADE + -A KUBE-MISC-TWO -j ACCEPT + COMMIT + *filter + :INPUT - [0:0] + :FORWARD - [0:0] + :OUTPUT - [0:0] + COMMIT + *mangle + COMMIT + `, "\n")) + if buf.String() != expected { + t.Fatalf("bad post-restore dump. expected:\n%s\n\ngot:\n%s\n", expected, buf.Bytes()) + } + + // Trying to use Restore to delete a chain that another chain jumps to will fail + rules = dedent.Dedent(strings.Trim(` + *nat + :KUBE-MISC-TWO - [0:0] + -X KUBE-MISC-TWO + COMMIT + `, "\n")) + err = fake.Restore(iptables.TableNAT, []byte(rules), iptables.NoFlushTables, iptables.RestoreCounters) + if err == nil || !strings.Contains(err.Error(), "referenced by existing rules") { + t.Fatalf("Expected 'referenced by existing rules' error from Restore, got %v", err) + } + + // Trying to use Restore to add a jump to a non-existent chain will fail + rules = dedent.Dedent(strings.Trim(` + *nat + :KUBE-MISC-TWO - [0:0] + -A KUBE-MISC-TWO -j KUBE-MISC-THREE + COMMIT + `, "\n")) + err = fake.Restore(iptables.TableNAT, []byte(rules), iptables.NoFlushTables, iptables.RestoreCounters) + if err == nil || !strings.Contains(err.Error(), "non-existent chain") { + t.Fatalf("Expected 'non-existent chain' error from Restore, got %v", err) + } + + // more Restore; empty out one chain and delete another, but also update its counters + rules = dedent.Dedent(strings.Trim(` + *nat + :KUBE-RESTORED - [0:0] + :KUBE-TEST - [99:9999] + -X KUBE-RESTORED + COMMIT + `, "\n")) + err = fake.Restore(iptables.TableNAT, []byte(rules), iptables.NoFlushTables, iptables.RestoreCounters) + if err != nil { + t.Fatalf("unexpected error from Restore: %v", err) + } + + buf.Reset() + err = fake.SaveInto("", buf) + if err != nil { + t.Fatalf("unexpected error from SaveInto: %v", err) + } + expected = dedent.Dedent(strings.Trim(` + *nat + :PREROUTING - [0:0] + :INPUT - [0:0] + :OUTPUT - [0:0] + :POSTROUTING - [0:0] + :KUBE-TEST - [99:9999] + :KUBE-MISC-CHAIN - [0:0] + :KUBE-MISC-TWO - [0:0] + :KUBE-EMPTY - [0:0] + -A KUBE-MISC-CHAIN -s 1.2.3.4 -j KUBE-MISC-TWO + -A KUBE-MISC-CHAIN -d 5.6.7.8 -j MASQUERADE + -A KUBE-MISC-TWO -j ACCEPT + COMMIT + *filter + :INPUT - [0:0] + :FORWARD - [0:0] + :OUTPUT - [0:0] + COMMIT + *mangle + COMMIT + `, "\n")) + if buf.String() != expected { + t.Fatalf("bad post-second-restore dump. expected:\n%s\n\ngot:\n%s\n", expected, buf.Bytes()) + } + + // RestoreAll, FlushTables + rules = dedent.Dedent(strings.Trim(` + *filter + :INPUT - [0:0] + :FORWARD - [0:0] + :OUTPUT - [0:0] + :KUBE-TEST - [0:0] + -A KUBE-TEST -m comment --comment "filter table KUBE-TEST" -j ACCEPT + COMMIT + *nat + :PREROUTING - [0:0] + :INPUT - [0:0] + :OUTPUT - [0:0] + :POSTROUTING - [0:0] + :KUBE-TEST - [88:8888] + :KUBE-NEW-CHAIN - [0:0] + -A KUBE-NEW-CHAIN -d 172.30.0.1 -j DNAT --to-destination 10.0.0.1 + -A KUBE-NEW-CHAIN -d 172.30.0.2 -j DNAT --to-destination 10.0.0.2 + -A KUBE-NEW-CHAIN -d 172.30.0.3 -j DNAT --to-destination 10.0.0.3 + COMMIT + `, "\n")) + err = fake.RestoreAll([]byte(rules), iptables.FlushTables, iptables.NoRestoreCounters) + if err != nil { + t.Fatalf("unexpected error from RestoreAll: %v", err) + } + + buf.Reset() + err = fake.SaveInto("", buf) + if err != nil { + t.Fatalf("unexpected error from SaveInto: %v", err) + } + expected = dedent.Dedent(strings.Trim(` + *nat + :PREROUTING - [0:0] + :INPUT - [0:0] + :OUTPUT - [0:0] + :POSTROUTING - [0:0] + :KUBE-TEST - [88:8888] + :KUBE-NEW-CHAIN - [0:0] + -A KUBE-NEW-CHAIN -d 172.30.0.1 -j DNAT --to-destination 10.0.0.1 + -A KUBE-NEW-CHAIN -d 172.30.0.2 -j DNAT --to-destination 10.0.0.2 + -A KUBE-NEW-CHAIN -d 172.30.0.3 -j DNAT --to-destination 10.0.0.3 + COMMIT + *filter + :INPUT - [0:0] + :FORWARD - [0:0] + :OUTPUT - [0:0] + :KUBE-TEST - [0:0] + -A KUBE-TEST -m comment --comment "filter table KUBE-TEST" -j ACCEPT + COMMIT + *mangle + COMMIT + `, "\n")) + if buf.String() != expected { + t.Fatalf("bad post-restore-all dump. expected:\n%s\n\ngot:\n%s\n", expected, buf.Bytes()) } } diff --git a/pkg/util/iptables/testing/parse.go b/pkg/util/iptables/testing/parse.go new file mode 100644 index 00000000000..42df8195336 --- /dev/null +++ b/pkg/util/iptables/testing/parse.go @@ -0,0 +1,375 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "fmt" + "reflect" + "regexp" + "strconv" + "strings" + + "github.com/openyurtio/openyurt/pkg/util/iptables" +) + +// IPTablesDump represents a parsed IPTables rules dump (ie, the output of +// "iptables-save" or input to "iptables-restore") +type IPTablesDump struct { + Tables []Table +} + +// Table represents an IPTables table +type Table struct { + Name iptables.Table + Chains []Chain +} + +// Chain represents an IPTables chain +type Chain struct { + Name iptables.Chain + Packets uint64 + Bytes uint64 + Rules []*Rule + + // Deleted is set if the input contained a "-X Name" line; this would never + // appear in iptables-save output but it could appear in iptables-restore *input*. + Deleted bool +} + +var declareTableRegex = regexp.MustCompile(`^\*(.*)$`) +var declareChainRegex = regexp.MustCompile(`^:([^ ]*) - \[([0-9]*):([0-9]*)\]$`) +var addRuleRegex = regexp.MustCompile(`^-A ([^ ]*) (.*)$`) +var deleteChainRegex = regexp.MustCompile(`^-X (.*)$`) + +type parseState int + +const ( + parseTableDeclaration parseState = iota + parseChainDeclarations + parseChains +) + +// ParseIPTablesDump parses an IPTables rules dump. Note: this may ignore some bad data. +func ParseIPTablesDump(data string) (*IPTablesDump, error) { + dump := &IPTablesDump{} + state := parseTableDeclaration + lines := strings.Split(strings.Trim(data, "\n"), "\n") + var t *Table + + for _, line := range lines { + retry: + line = strings.TrimSpace(line) + if line == "" || line[0] == '#' { + continue + } + + switch state { + case parseTableDeclaration: + // Parse table declaration line ("*filter"). + match := declareTableRegex.FindStringSubmatch(line) + if match == nil { + return nil, fmt.Errorf("could not parse iptables data (table %d starts with %q not a table name)", len(dump.Tables)+1, line) + } + dump.Tables = append(dump.Tables, Table{Name: iptables.Table(match[1])}) + t = &dump.Tables[len(dump.Tables)-1] + state = parseChainDeclarations + + case parseChainDeclarations: + match := declareChainRegex.FindStringSubmatch(line) + if match == nil { + state = parseChains + goto retry + } + + chain := iptables.Chain(match[1]) + packets, _ := strconv.ParseUint(match[2], 10, 64) + bytes, _ := strconv.ParseUint(match[3], 10, 64) + + t.Chains = append(t.Chains, + Chain{ + Name: chain, + Packets: packets, + Bytes: bytes, + }, + ) + + case parseChains: + if match := addRuleRegex.FindStringSubmatch(line); match != nil { + chain := iptables.Chain(match[1]) + + c, err := dump.GetChain(t.Name, chain) + if err != nil { + return nil, fmt.Errorf("error parsing rule %q: %v", line, err) + } + if c.Deleted { + return nil, fmt.Errorf("cannot add rules to deleted chain %q", chain) + } + + rule, err := ParseRule(line, false) + if err != nil { + return nil, err + } + c.Rules = append(c.Rules, rule) + } else if match := deleteChainRegex.FindStringSubmatch(line); match != nil { + chain := iptables.Chain(match[1]) + + c, err := dump.GetChain(t.Name, chain) + if err != nil { + return nil, fmt.Errorf("error parsing rule %q: %v", line, err) + } + if len(c.Rules) != 0 { + return nil, fmt.Errorf("cannot delete chain %q after adding rules", chain) + } + c.Deleted = true + } else if line == "COMMIT" { + state = parseTableDeclaration + } else { + return nil, fmt.Errorf("error parsing rule %q", line) + } + } + } + + if state != parseTableDeclaration { + return nil, fmt.Errorf("could not parse iptables data (no COMMIT line?)") + } + + return dump, nil +} + +func (dump *IPTablesDump) String() string { + buffer := &strings.Builder{} + for _, t := range dump.Tables { + fmt.Fprintf(buffer, "*%s\n", t.Name) + for _, c := range t.Chains { + fmt.Fprintf(buffer, ":%s - [%d:%d]\n", c.Name, c.Packets, c.Bytes) + } + for _, c := range t.Chains { + for _, r := range c.Rules { + fmt.Fprintf(buffer, "%s\n", r.Raw) + } + } + for _, c := range t.Chains { + if c.Deleted { + fmt.Fprintf(buffer, "-X %s\n", c.Name) + } + } + fmt.Fprintf(buffer, "COMMIT\n") + } + return buffer.String() +} + +func (dump *IPTablesDump) GetTable(table iptables.Table) (*Table, error) { + for i := range dump.Tables { + if dump.Tables[i].Name == table { + return &dump.Tables[i], nil + } + } + return nil, fmt.Errorf("no such table %q", table) +} + +func (dump *IPTablesDump) GetChain(table iptables.Table, chain iptables.Chain) (*Chain, error) { + t, err := dump.GetTable(table) + if err != nil { + return nil, err + } + for i := range t.Chains { + if t.Chains[i].Name == chain { + return &t.Chains[i], nil + } + } + return nil, fmt.Errorf("no such chain %q", chain) +} + +// Rule represents a single parsed IPTables rule. (This currently covers all of the rule +// types that we actually use in pkg/proxy/iptables or pkg/proxy/ipvs.) +// +// The parsing is mostly-automated based on type reflection. The `param` tag on a field +// indicates the parameter whose value will be placed into that field. (The code assumes +// that we don't use both the short and long forms of any parameter names (eg, "-s" vs +// "--source"), which is currently true, but it could be extended if necessary.) The +// `negatable` tag indicates if a parameter is allowed to be preceded by "!". +// +// Parameters that take a value are stored as type `*IPTablesValue`, which encapsulates a +// string value and whether the rule was negated (ie, whether the rule requires that we +// *match* or *don't match* that value). But string-valued parameters that can't be +// negated use `IPTablesValue` rather than `string` too, just for API consistency. +// +// Parameters that don't take a value are stored as `*bool`, where the value is `nil` if +// the parameter was not present, `&true` if the parameter was present, or `&false` if the +// parameter was present but negated. +// +// Parsing skips over "-m MODULE" parameters because most parameters have unique names +// anyway even ignoring the module name, and in the cases where they don't (eg "-m tcp +// --sport" vs "-m udp --sport") the parameters are mutually-exclusive and it's more +// convenient to store them in the same struct field anyway. +type Rule struct { + // Raw contains the original raw rule string + Raw string + + Chain iptables.Chain `param:"-A"` + Comment *IPTablesValue `param:"--comment"` + + Protocol *IPTablesValue `param:"-p" negatable:"true"` + + SourceAddress *IPTablesValue `param:"-s" negatable:"true"` + SourceType *IPTablesValue `param:"--src-type" negatable:"true"` + SourcePort *IPTablesValue `param:"--sport" negatable:"true"` + + DestinationAddress *IPTablesValue `param:"-d" negatable:"true"` + DestinationType *IPTablesValue `param:"--dst-type" negatable:"true"` + DestinationPort *IPTablesValue `param:"--dport" negatable:"true"` + + MatchSet *IPTablesValue `param:"--match-set" negatable:"true"` + + Jump *IPTablesValue `param:"-j"` + RandomFully *bool `param:"--random-fully"` + Probability *IPTablesValue `param:"--probability"` + DNATDestination *IPTablesValue `param:"--to-destination"` + + // We don't actually use the values of these, but we care if they are present + AffinityCheck *bool `param:"--rcheck" negatable:"true"` + MarkCheck *IPTablesValue `param:"--mark" negatable:"true"` + CTStateCheck *IPTablesValue `param:"--ctstate" negatable:"true"` + + // We don't currently care about any of these in the unit tests, but we expect + // them to be present in some rules that we parse, so we define how to parse them. + AffinityName *IPTablesValue `param:"--name"` + AffinitySeconds *IPTablesValue `param:"--seconds"` + AffinitySet *bool `param:"--set" negatable:"true"` + AffinityReap *bool `param:"--reap"` + StatisticMode *IPTablesValue `param:"--mode"` +} + +// IPTablesValue is a value of a parameter in an Rule, where the parameter is +// possibly negated. +type IPTablesValue struct { + Negated bool + Value string +} + +// for debugging; otherwise %v will just print the pointer value +func (v *IPTablesValue) String() string { + if v.Negated { + return fmt.Sprintf("NOT %q", v.Value) + } else { + return fmt.Sprintf("%q", v.Value) + } +} + +// Matches returns true if cmp equals / doesn't equal v.Value (depending on +// v.Negated). +func (v *IPTablesValue) Matches(cmp string) bool { + if v.Negated { + return v.Value != cmp + } else { + return v.Value == cmp + } +} + +// findParamField finds a field in value with the struct tag "param:${param}" and if found, +// returns a pointer to the Value of that field, and the value of its "negatable" tag. +func findParamField(value reflect.Value, param string) (*reflect.Value, bool) { + typ := value.Type() + for i := 0; i < typ.NumField(); i++ { + field := typ.Field(i) + if field.Tag.Get("param") == param { + fValue := value.Field(i) + return &fValue, field.Tag.Get("negatable") == "true" + } + } + return nil, false +} + +// wordRegex matches a single word or a quoted string (at the start of the string, or +// preceded by whitespace) +var wordRegex = regexp.MustCompile(`(?:^|\s)("[^"]*"|[^"]\S*)`) + +// Used by ParseRule +var boolPtrType = reflect.PointerTo(reflect.TypeOf(true)) +var ipTablesValuePtrType = reflect.TypeOf((*IPTablesValue)(nil)) + +// ParseRule parses rule. If strict is false, it will parse the recognized +// parameters and ignore unrecognized ones. If it is true, parsing will fail if there are +// unrecognized parameters. +func ParseRule(rule string, strict bool) (*Rule, error) { + parsed := &Rule{Raw: rule} + + // Split rule into "words" (where a quoted string is a single word). + var words []string + for _, match := range wordRegex.FindAllStringSubmatch(rule, -1) { + words = append(words, strings.Trim(match[1], `"`)) + } + + // The chain name must come first (and can't be the only thing there) + if len(words) < 2 || words[0] != "-A" { + return nil, fmt.Errorf(`bad iptables rule (does not start with "-A CHAIN")`) + } else if len(words) < 3 { + return nil, fmt.Errorf("bad iptables rule (no match rules)") + } + + // For each word, see if it is a known iptables parameter, based on the struct + // field tags in Rule. Note that in the non-strict case we implicitly assume that + // no unrecognized parameter will take an argument that could be mistaken for + // another parameter. + v := reflect.ValueOf(parsed).Elem() + negated := false + for w := 0; w < len(words); { + if words[w] == "-m" && w < len(words)-1 { + // Skip "-m MODULE"; we don't pay attention to that since the + // parameter names are unique enough anyway. + w += 2 + continue + } + + if words[w] == "!" { + negated = true + w++ + continue + } + + // For everything else, see if it corresponds to a field of Rule + if field, negatable := findParamField(v, words[w]); field != nil { + if negated && !negatable { + return nil, fmt.Errorf("cannot negate parameter %q", words[w]) + } + if field.Type() != boolPtrType && w == len(words)-1 { + return nil, fmt.Errorf("parameter %q requires an argument", words[w]) + } + switch field.Type() { + case boolPtrType: + boolVal := !negated + field.Set(reflect.ValueOf(&boolVal)) + w++ + case ipTablesValuePtrType: + field.Set(reflect.ValueOf(&IPTablesValue{Negated: negated, Value: words[w+1]})) + w += 2 + default: + field.SetString(words[w+1]) + w += 2 + } + } else if strict { + return nil, fmt.Errorf("unrecognized parameter %q", words[w]) + } else { + // skip + w++ + } + + negated = false + } + + return parsed, nil +} diff --git a/pkg/util/iptables/testing/parse_test.go b/pkg/util/iptables/testing/parse_test.go new file mode 100644 index 00000000000..6a33d2aaf3c --- /dev/null +++ b/pkg/util/iptables/testing/parse_test.go @@ -0,0 +1,595 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "fmt" + "reflect" + "strings" + "testing" + + "github.com/lithammer/dedent" + utilpointer "k8s.io/utils/pointer" + + "github.com/openyurtio/openyurt/pkg/util/iptables" +) + +func TestParseRule(t *testing.T) { + testCases := []struct { + name string + rule string + parsed *Rule + nonStrict bool + err string + }{ + { + name: "basic rule", + rule: `-A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT`, + parsed: &Rule{ + Raw: `-A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT`, + Chain: iptables.Chain("KUBE-NODEPORTS"), + Comment: &IPTablesValue{Value: "ns2/svc2:p80 health check node port"}, + Protocol: &IPTablesValue{Value: "tcp"}, + DestinationPort: &IPTablesValue{Value: "30000"}, + Jump: &IPTablesValue{Value: "ACCEPT"}, + }, + }, + { + name: "addRuleToChainRegex requires an actual rule, not just a chain name", + rule: `-A KUBE-NODEPORTS`, + err: `(no match rules)`, + }, + { + name: "ParseRule only parses adds", + rule: `-D KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT`, + err: `(does not start with "-A CHAIN")`, + }, + { + name: "unquoted comment", + rule: `-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ`, + parsed: &Rule{ + Raw: `-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ`, + Chain: iptables.Chain("KUBE-SVC-XPGD46QRK7WJZT7O"), + Comment: &IPTablesValue{Value: "ns1/svc1:p80"}, + Jump: &IPTablesValue{Value: "KUBE-SEP-SXIVWICOYRO3J4NJ"}, + }, + }, + { + name: "local source", + rule: `-A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "masquerade LOCAL traffic for ns2/svc2:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ`, + parsed: &Rule{ + Raw: `-A KUBE-XLB-GNZBNJ2PO5MGZ6GT -m comment --comment "masquerade LOCAL traffic for ns2/svc2:p80 LB IP" -m addrtype --src-type LOCAL -j KUBE-MARK-MASQ`, + Chain: iptables.Chain("KUBE-XLB-GNZBNJ2PO5MGZ6GT"), + Comment: &IPTablesValue{Value: "masquerade LOCAL traffic for ns2/svc2:p80 LB IP"}, + SourceType: &IPTablesValue{Value: "LOCAL"}, + Jump: &IPTablesValue{Value: "KUBE-MARK-MASQ"}, + }, + }, + { + name: "not local destination", + rule: `-A RULE-TYPE-NOT-CURRENTLY-USED-BY-KUBE-PROXY -m addrtype ! --dst-type LOCAL -j KUBE-MARK-MASQ`, + parsed: &Rule{ + Raw: `-A RULE-TYPE-NOT-CURRENTLY-USED-BY-KUBE-PROXY -m addrtype ! --dst-type LOCAL -j KUBE-MARK-MASQ`, + Chain: iptables.Chain("RULE-TYPE-NOT-CURRENTLY-USED-BY-KUBE-PROXY"), + DestinationType: &IPTablesValue{Negated: true, Value: "LOCAL"}, + Jump: &IPTablesValue{Value: "KUBE-MARK-MASQ"}, + }, + }, + { + name: "destination IP/port", + rule: `-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O`, + parsed: &Rule{ + Raw: `-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O`, + Chain: iptables.Chain("KUBE-SERVICES"), + Comment: &IPTablesValue{Value: "ns1/svc1:p80 cluster IP"}, + Protocol: &IPTablesValue{Value: "tcp"}, + DestinationAddress: &IPTablesValue{Value: "172.30.0.41"}, + DestinationPort: &IPTablesValue{Value: "80"}, + Jump: &IPTablesValue{Value: "KUBE-SVC-XPGD46QRK7WJZT7O"}, + }, + }, + { + name: "source IP", + rule: `-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ`, + parsed: &Rule{ + Raw: `-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ`, + Chain: iptables.Chain("KUBE-SEP-SXIVWICOYRO3J4NJ"), + Comment: &IPTablesValue{Value: "ns1/svc1:p80"}, + SourceAddress: &IPTablesValue{Value: "10.180.0.1"}, + Jump: &IPTablesValue{Value: "KUBE-MARK-MASQ"}, + }, + }, + { + name: "not source IP", + rule: `-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ`, + parsed: &Rule{ + Raw: `-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 172.30.0.41 --dport 80 ! -s 10.0.0.0/8 -j KUBE-MARK-MASQ`, + Chain: iptables.Chain("KUBE-SVC-XPGD46QRK7WJZT7O"), + Comment: &IPTablesValue{Value: "ns1/svc1:p80 cluster IP"}, + Protocol: &IPTablesValue{Value: "tcp"}, + DestinationAddress: &IPTablesValue{Value: "172.30.0.41"}, + DestinationPort: &IPTablesValue{Value: "80"}, + SourceAddress: &IPTablesValue{Negated: true, Value: "10.0.0.0/8"}, + Jump: &IPTablesValue{Value: "KUBE-MARK-MASQ"}, + }, + }, + { + name: "affinity", + rule: `-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m recent --name KUBE-SEP-SXIVWICOYRO3J4NJ --rcheck --seconds 10800 --reap -j KUBE-SEP-SXIVWICOYRO3J4NJ`, + parsed: &Rule{ + Raw: `-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -m recent --name KUBE-SEP-SXIVWICOYRO3J4NJ --rcheck --seconds 10800 --reap -j KUBE-SEP-SXIVWICOYRO3J4NJ`, + Chain: iptables.Chain("KUBE-SVC-XPGD46QRK7WJZT7O"), + Comment: &IPTablesValue{Value: "ns1/svc1:p80"}, + AffinityName: &IPTablesValue{Value: "KUBE-SEP-SXIVWICOYRO3J4NJ"}, + AffinitySeconds: &IPTablesValue{Value: "10800"}, + AffinityCheck: utilpointer.Bool(true), + AffinityReap: utilpointer.Bool(true), + Jump: &IPTablesValue{Value: "KUBE-SEP-SXIVWICOYRO3J4NJ"}, + }, + }, + { + name: "jump to DNAT", + rule: `-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80`, + parsed: &Rule{ + Raw: `-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80`, + Chain: iptables.Chain("KUBE-SEP-SXIVWICOYRO3J4NJ"), + Comment: &IPTablesValue{Value: "ns1/svc1:p80"}, + Protocol: &IPTablesValue{Value: "tcp"}, + Jump: &IPTablesValue{Value: "DNAT"}, + DNATDestination: &IPTablesValue{Value: "10.180.0.1:80"}, + }, + }, + { + name: "jump to endpoint", + rule: `-A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-UKSFD7AGPMPPLUHC`, + parsed: &Rule{ + Raw: `-A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-UKSFD7AGPMPPLUHC`, + Chain: iptables.Chain("KUBE-SVC-4SW47YFZTEDKD3PK"), + Comment: &IPTablesValue{Value: "ns4/svc4:p80"}, + Probability: &IPTablesValue{Value: "0.5000000000"}, + StatisticMode: &IPTablesValue{Value: "random"}, + Jump: &IPTablesValue{Value: "KUBE-SEP-UKSFD7AGPMPPLUHC"}, + }, + }, + { + name: "unrecognized arguments", + rule: `-A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -i eth0 -j KUBE-SEP-UKSFD7AGPMPPLUHC`, + err: `unrecognized parameter "-i"`, + }, + { + name: "unrecognized arguments with strict=false", + rule: `-A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -i eth0 -j KUBE-SEP-UKSFD7AGPMPPLUHC`, + nonStrict: true, + parsed: &Rule{ + Raw: `-A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -i eth0 -j KUBE-SEP-UKSFD7AGPMPPLUHC`, + Chain: iptables.Chain("KUBE-SVC-4SW47YFZTEDKD3PK"), + Comment: &IPTablesValue{Value: "ns4/svc4:p80"}, + Jump: &IPTablesValue{Value: "KUBE-SEP-UKSFD7AGPMPPLUHC"}, + }, + }, + { + name: "bad use of !", + rule: `-A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 ! -j KUBE-SEP-UKSFD7AGPMPPLUHC`, + err: `cannot negate parameter "-j"`, + }, + { + name: "missing argument", + rule: `-A KUBE-SVC-4SW47YFZTEDKD3PK -m comment --comment ns4/svc4:p80 -j`, + err: `parameter "-j" requires an argument`, + }, + { + name: "negated bool arg", + rule: `-A TEST -m recent ! --rcheck -j KUBE-SEP-SXIVWICOYRO3J4NJ`, + parsed: &Rule{ + Raw: `-A TEST -m recent ! --rcheck -j KUBE-SEP-SXIVWICOYRO3J4NJ`, + Chain: iptables.Chain("TEST"), + AffinityCheck: utilpointer.Bool(false), + Jump: &IPTablesValue{Value: "KUBE-SEP-SXIVWICOYRO3J4NJ"}, + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + rule, err := ParseRule(testCase.rule, !testCase.nonStrict) + if err != nil { + if testCase.err == "" { + t.Errorf("expected %+v, got error %q", testCase.parsed, err) + } else if !strings.Contains(err.Error(), testCase.err) { + t.Errorf("wrong error, expected %q got %q", testCase.err, err) + } + } else { + if testCase.err != "" { + t.Errorf("expected error %q, got %+v", testCase.err, rule) + } else if !reflect.DeepEqual(rule, testCase.parsed) { + t.Errorf("bad match: expected\n%+v\ngot\n%+v", testCase.parsed, rule) + } + } + }) + } +} + +// Helper for TestParseIPTablesDump. Obviously it should not be used in TestParseRule... +func mustParseRule(rule string) *Rule { + parsed, err := ParseRule(rule, false) + if err != nil { + panic(fmt.Sprintf("failed to parse test case rule %q: %v", rule, err)) + } + return parsed +} + +func TestParseIPTablesDump(t *testing.T) { + for _, tc := range []struct { + name string + input string + output *IPTablesDump + error string + }{ + { + name: "basic test", + input: dedent.Dedent(` + *filter + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-SERVICES - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + COMMIT + `), + output: &IPTablesDump{ + Tables: []Table{{ + Name: iptables.TableFilter, + Chains: []Chain{{ + Name: iptables.Chain("KUBE-SERVICES"), + }, { + Name: iptables.Chain("KUBE-EXTERNAL-SERVICES"), + }, { + Name: iptables.Chain("KUBE-FORWARD"), + Rules: []*Rule{ + mustParseRule(`-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP`), + mustParseRule(`-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT`), + mustParseRule(`-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT`), + }, + }, { + Name: iptables.Chain("KUBE-NODEPORTS"), + Rules: []*Rule{ + mustParseRule(`-A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT`), + }, + }}, + }, { + Name: iptables.TableNAT, + Chains: []Chain{{ + Name: iptables.Chain("KUBE-SERVICES"), + Rules: []*Rule{ + mustParseRule(`-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O`), + mustParseRule(`-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS`), + }, + }, { + Name: iptables.Chain("KUBE-NODEPORTS"), + }, { + Name: iptables.Chain("KUBE-POSTROUTING"), + Rules: []*Rule{ + mustParseRule(`-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN`), + mustParseRule(`-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000`), + mustParseRule(`-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE`), + }, + }, { + Name: iptables.Chain("KUBE-MARK-MASQ"), + Rules: []*Rule{ + mustParseRule(`-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000`), + }, + }, { + Name: iptables.Chain("KUBE-SVC-XPGD46QRK7WJZT7O"), + Rules: []*Rule{ + mustParseRule(`-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ`), + mustParseRule(`-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ`), + }, + }, { + Name: iptables.Chain("KUBE-SEP-SXIVWICOYRO3J4NJ"), + Rules: []*Rule{ + mustParseRule(`-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ`), + mustParseRule(`-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80`), + }, + }}, + }}, + }, + }, + { + name: "deletion", + input: dedent.Dedent(` + *nat + :KUBE-SERVICES - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] + -X KUBE-SVC-XPGD46QRK7WJZT7O + -X KUBE-SEP-SXIVWICOYRO3J4NJ + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + COMMIT + `), + output: &IPTablesDump{ + Tables: []Table{{ + Name: iptables.TableNAT, + Chains: []Chain{{ + Name: iptables.Chain("KUBE-SERVICES"), + Rules: []*Rule{ + mustParseRule(`-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS`), + }, + }, { + Name: iptables.Chain("KUBE-SVC-XPGD46QRK7WJZT7O"), + Deleted: true, + }, { + Name: iptables.Chain("KUBE-SEP-SXIVWICOYRO3J4NJ"), + Deleted: true, + }}, + }}, + }, + }, + { + name: "whitespace and comments", + input: dedent.Dedent(` + # Generated by iptables-save v1.8.7 on Mon May 9 11:22:21 2022 + # (not really...) + *filter + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + # This rule does a thing + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + # Completed on Mon May 9 11:22:21 2022 + `), + output: &IPTablesDump{ + Tables: []Table{{ + Name: iptables.TableFilter, + Chains: []Chain{{ + Name: iptables.Chain("KUBE-SERVICES"), + }, { + Name: iptables.Chain("KUBE-EXTERNAL-SERVICES"), + }, { + Name: iptables.Chain("KUBE-FORWARD"), + Rules: []*Rule{ + mustParseRule(`-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP`), + mustParseRule(`-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT`), + mustParseRule(`-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT`), + }, + }, { + Name: iptables.Chain("KUBE-NODEPORTS"), + Rules: []*Rule{ + mustParseRule(`-A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT`), + }, + }}, + }}, + }, + }, + { + name: "no COMMIT line", + input: dedent.Dedent(` + *filter + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + `), + error: "no COMMIT line?", + }, + { + name: "two tables, no second COMMIT line", + input: dedent.Dedent(` + *filter + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-SERVICES - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + `), + error: "no COMMIT line?", + }, + { + name: "two tables, no second header line", + input: dedent.Dedent(` + *filter + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + :KUBE-SERVICES - [0:0] + :KUBE-NODEPORTS - [0:0] + :KUBE-POSTROUTING - [0:0] + :KUBE-MARK-MASQ - [0:0] + :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] + :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] + -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN + -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 + -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE + -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 + -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ + -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ + -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 + -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS + COMMIT + `), + error: "not a table name", + }, + { + name: "trailing junk", + input: dedent.Dedent(` + *filter + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + *nat + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + junk + `), + error: `table 3 starts with "junk"`, + }, + { + name: "add to missing chain", + input: dedent.Dedent(` + *filter + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + `), + error: `no such chain "KUBE-FORWARD"`, + }, + { + name: "add to deleted chain", + input: dedent.Dedent(` + *filter + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -X KUBE-FORWARD + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + `), + error: `cannot add rules to deleted chain`, + }, + { + name: "deleted non-empty chain", + input: dedent.Dedent(` + *filter + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + -X KUBE-FORWARD + COMMIT + `), + error: `cannot delete chain "KUBE-FORWARD" after adding rules`, + }, + { + name: "junk rule", + input: dedent.Dedent(` + *filter + :KUBE-SERVICES - [0:0] + :KUBE-EXTERNAL-SERVICES - [0:0] + :KUBE-FORWARD - [0:0] + :KUBE-NODEPORTS - [0:0] + -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT + -Q KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT + -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + COMMIT + `), + error: `"-Q KUBE-FORWARD`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + dump, err := ParseIPTablesDump(tc.input) + if err == nil { + if tc.error != "" { + t.Errorf("unexpectedly did not get error") + } else if !reflect.DeepEqual(tc.output, dump) { + t.Errorf("bad output: expected %#v got %#v", tc.output, dump) + } + } else { + if tc.error == "" { + t.Errorf("got unexpected error: %v", err) + } else if !strings.Contains(err.Error(), tc.error) { + t.Errorf("got wrong error: %v (expected %q)", err, tc.error) + } + } + }) + } +} diff --git a/pkg/util/kubernetes/kubeadm/app/util/apiclient/idempotency.go b/pkg/util/kubernetes/kubeadm/app/util/apiclient/idempotency.go index 4fecdde5bb7..3fd7b65403d 100644 --- a/pkg/util/kubernetes/kubeadm/app/util/apiclient/idempotency.go +++ b/pkg/util/kubernetes/kubeadm/app/util/apiclient/idempotency.go @@ -73,7 +73,7 @@ func CreateOrUpdateSecret(client clientset.Interface, secret *v1.Secret) error { // CreateOrUpdateRole creates a Role if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. func CreateOrUpdateRole(client clientset.Interface, role *rbac.Role) error { var lastError error - err := wait.PollImmediate(constants.APICallRetryInterval, constants.APICallWithWriteTimeout, func() (bool, error) { + err := wait.PollUntilContextTimeout(context.Background(), constants.APICallRetryInterval, constants.APICallWithWriteTimeout, true, func(ctx context.Context) (bool, error) { if _, err := client.RbacV1().Roles(role.ObjectMeta.Namespace).Create(context.TODO(), role, metav1.CreateOptions{}); err != nil { if !apierrors.IsAlreadyExists(err) { lastError = errors.Wrap(err, "unable to create RBAC role") @@ -96,7 +96,7 @@ func CreateOrUpdateRole(client clientset.Interface, role *rbac.Role) error { // CreateOrUpdateRoleBinding creates a RoleBinding if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. func CreateOrUpdateRoleBinding(client clientset.Interface, roleBinding *rbac.RoleBinding) error { var lastError error - err := wait.PollImmediate(constants.APICallRetryInterval, constants.APICallWithWriteTimeout, func() (bool, error) { + err := wait.PollUntilContextTimeout(context.Background(), constants.APICallRetryInterval, constants.APICallWithWriteTimeout, true, func(ctx context.Context) (bool, error) { if _, err := client.RbacV1().RoleBindings(roleBinding.ObjectMeta.Namespace).Create(context.TODO(), roleBinding, metav1.CreateOptions{}); err != nil { if !apierrors.IsAlreadyExists(err) { lastError = errors.Wrap(err, "unable to create RBAC rolebinding") diff --git a/pkg/util/taints/taints.go b/pkg/util/taints/taints.go index 942743a1e8c..6a2c953b943 100644 --- a/pkg/util/taints/taints.go +++ b/pkg/util/taints/taints.go @@ -28,12 +28,6 @@ import ( "github.com/openyurtio/openyurt/pkg/util/helper" ) -const ( - MODIFIED = "modified" - TAINTED = "tainted" - UNTAINTED = "untainted" -) - // parseTaint parses a taint from a string, whose form must be either // '=:', ':', or ''. func parseTaint(st string) (v1.Taint, error) { @@ -91,7 +85,7 @@ func validateTaintEffect(effect v1.TaintEffect) error { // It also validates the spec. For example, the form `` may be used to remove a taint, but not to add one. func ParseTaints(spec []string) ([]v1.Taint, []v1.Taint, error) { var taints, taintsToRemove []v1.Taint - uniqueTaints := map[v1.TaintEffect]sets.String{} + uniqueTaints := map[v1.TaintEffect]sets.Set[string]{} for _, taintSpec := range spec { if strings.HasSuffix(taintSpec, "-") { @@ -115,7 +109,7 @@ func ParseTaints(spec []string) ([]v1.Taint, []v1.Taint, error) { } // add taint to existingTaints for uniqueness check if len(uniqueTaints[newTaint.Effect]) == 0 { - uniqueTaints[newTaint.Effect] = sets.String{} + uniqueTaints[newTaint.Effect] = sets.Set[string]{} } uniqueTaints[newTaint.Effect].Insert(newTaint.Key) diff --git a/pkg/yurtadm/cmd/join/join.go b/pkg/yurtadm/cmd/join/join.go index d05b9323e0a..ee5285f9200 100644 --- a/pkg/yurtadm/cmd/join/join.go +++ b/pkg/yurtadm/cmd/join/join.go @@ -223,7 +223,7 @@ type joinData struct { token string tlsBootstrapCfg *clientcmdapi.Config clientSet *clientset.Clientset - ignorePreflightErrors sets.String + ignorePreflightErrors sets.Set[string] organizations string pauseImage string yurthubImage string @@ -275,7 +275,7 @@ func newJoinData(args []string, opt *joinOptions) (*joinData, error) { return nil, errors.Errorf("when --discovery-token-ca-cert-hash is not specified, --discovery-token-unsafe-skip-ca-verification should be true") } - ignoreErrors := sets.String{} + ignoreErrors := sets.Set[string]{} for i := range opt.ignorePreflightErrors { ignoreErrors.Insert(opt.ignorePreflightErrors[i]) } @@ -473,7 +473,7 @@ func (j *joinData) NodeRegistration() *joindata.NodeRegistration { } // IgnorePreflightErrors returns the list of preflight errors to ignore. -func (j *joinData) IgnorePreflightErrors() sets.String { +func (j *joinData) IgnorePreflightErrors() sets.Set[string] { return j.ignorePreflightErrors } diff --git a/pkg/yurtadm/cmd/join/join_test.go b/pkg/yurtadm/cmd/join/join_test.go index 29fe42ad1d5..5c47f6636b9 100644 --- a/pkg/yurtadm/cmd/join/join_test.go +++ b/pkg/yurtadm/cmd/join/join_test.go @@ -529,16 +529,16 @@ func TestNodeRegistration(t *testing.T) { func TestIgnorePreflightErrors(t *testing.T) { jd := joinData{ - ignorePreflightErrors: sets.String{}, + ignorePreflightErrors: sets.Set[string]{}, } tests := []struct { name string - expect sets.String + expect sets.Set[string] }{ { "normal", - sets.String{}, + sets.Set[string]{}, }, } diff --git a/pkg/yurtadm/cmd/join/joindata/data.go b/pkg/yurtadm/cmd/join/joindata/data.go index 80b1ad015f6..98089565dc3 100644 --- a/pkg/yurtadm/cmd/join/joindata/data.go +++ b/pkg/yurtadm/cmd/join/joindata/data.go @@ -45,7 +45,7 @@ type YurtJoinData interface { NodeRegistration() *NodeRegistration CaCertHashes() []string NodeLabels() map[string]string - IgnorePreflightErrors() sets.String + IgnorePreflightErrors() sets.Set[string] KubernetesResourceServer() string ReuseCNIBin() bool Namespace() string diff --git a/pkg/yurtadm/util/kubernetes/kubernetes.go b/pkg/yurtadm/util/kubernetes/kubernetes.go index f254f54b74b..e379411827b 100644 --- a/pkg/yurtadm/util/kubernetes/kubernetes.go +++ b/pkg/yurtadm/util/kubernetes/kubernetes.go @@ -390,7 +390,7 @@ func SetKubeadmJoinConfig(data joindata.YurtJoinData) error { ctx := map[string]interface{}{ "kubeConfigPath": KubeadmJoinDiscoveryFilePath, "tlsBootstrapToken": data.JoinToken(), - "ignorePreflightErrors": data.IgnorePreflightErrors().List(), + "ignorePreflightErrors": data.IgnorePreflightErrors().UnsortedList(), "podInfraContainerImage": data.PauseImage(), "nodeLabels": constructNodeLabels(data.NodeLabels(), nodeReg.WorkingMode, projectinfo.GetEdgeWorkerLabelKey()), "criSocket": nodeReg.CRISocket, diff --git a/pkg/yurtadm/util/yurthub/yurthub.go b/pkg/yurtadm/util/yurthub/yurthub.go index c1ce55b8e8b..977d5e5e5af 100644 --- a/pkg/yurtadm/util/yurthub/yurthub.go +++ b/pkg/yurtadm/util/yurthub/yurthub.go @@ -19,6 +19,7 @@ package yurthub import ( "bufio" "bytes" + "context" "fmt" "io" "net/http" @@ -125,7 +126,7 @@ func CheckYurthubHealthz(yurthubServer string) error { return err } client := &http.Client{} - return wait.PollImmediate(time.Second*5, 300*time.Second, func() (bool, error) { + return wait.PollUntilContextTimeout(context.Background(), time.Second*5, 300*time.Second, true, func(ctx context.Context) (bool, error) { resp, err := client.Do(req) if err != nil { return false, nil @@ -145,7 +146,7 @@ func CheckYurthubReadyz(yurthubServer string) error { return err } client := &http.Client{} - return wait.PollImmediate(time.Second*5, 300*time.Second, func() (bool, error) { + return wait.PollUntilContextTimeout(context.Background(), time.Second*5, 300*time.Second, true, func(ctx context.Context) (bool, error) { resp, err := client.Do(req) if err != nil { return false, nil diff --git a/pkg/yurtadm/util/yurthub/yurthub_test.go b/pkg/yurtadm/util/yurthub/yurthub_test.go index aa3cf3bd87d..755fc456ce1 100644 --- a/pkg/yurtadm/util/yurthub/yurthub_test.go +++ b/pkg/yurtadm/util/yurthub/yurthub_test.go @@ -283,7 +283,7 @@ func (j *testData) NodeRegistration() *joindata.NodeRegistration { return j.joinNodeData } -func (j *testData) IgnorePreflightErrors() sets.String { +func (j *testData) IgnorePreflightErrors() sets.Set[string] { return nil } diff --git a/pkg/yurthub/cachemanager/cache_agent.go b/pkg/yurthub/cachemanager/cache_agent.go index 82fd9bb07a3..fd4d5728f03 100644 --- a/pkg/yurthub/cachemanager/cache_agent.go +++ b/pkg/yurthub/cachemanager/cache_agent.go @@ -35,13 +35,13 @@ const ( type CacheAgent struct { sync.Mutex - agents sets.String + agents sets.Set[string] store StorageWrapper } func NewCacheAgents(informerFactory informers.SharedInformerFactory, store StorageWrapper) *CacheAgent { ca := &CacheAgent{ - agents: sets.NewString(util.DefaultCacheAgents...), + agents: sets.New(util.DefaultCacheAgents...), store: store, } configmapInformer := informerFactory.Core().V1().ConfigMaps().Informer() @@ -99,8 +99,8 @@ func (ca *CacheAgent) deleteConfigmap(obj interface{}) { } // updateCacheAgents update cache agents -func (ca *CacheAgent) updateCacheAgents(cacheAgents, action string) sets.String { - newAgents := sets.NewString(util.DefaultCacheAgents...) +func (ca *CacheAgent) updateCacheAgents(cacheAgents, action string) sets.Set[string] { + newAgents := sets.New(util.DefaultCacheAgents...) for _, agent := range strings.Split(cacheAgents, sepForAgent) { agent = strings.TrimSpace(agent) if len(agent) != 0 { @@ -112,7 +112,7 @@ func (ca *CacheAgent) updateCacheAgents(cacheAgents, action string) sets.String defer ca.Unlock() if ca.agents.Equal(newAgents) { - return sets.String{} + return sets.Set[string]{} } // get deleted and added agents @@ -125,10 +125,10 @@ func (ca *CacheAgent) updateCacheAgents(cacheAgents, action string) sets.String return deletedAgents } -func (ca *CacheAgent) deleteAgentCache(deletedAgents sets.String) { +func (ca *CacheAgent) deleteAgentCache(deletedAgents sets.Set[string]) { // delete cache data for deleted agents if deletedAgents.Len() > 0 { - components := deletedAgents.List() + components := deletedAgents.UnsortedList() for i := range components { if err := ca.store.DeleteComponentResources(components[i]); err != nil { klog.Errorf("could not cleanup cache for deleted agent(%s), %v", components[i], err) diff --git a/pkg/yurthub/cachemanager/cache_agent_test.go b/pkg/yurthub/cachemanager/cache_agent_test.go index 1d2df14c72d..29016c2fe58 100644 --- a/pkg/yurthub/cachemanager/cache_agent_test.go +++ b/pkg/yurthub/cachemanager/cache_agent_test.go @@ -30,44 +30,44 @@ func TestUpdateCacheAgents(t *testing.T) { desc string initAgents []string cacheAgents string - resultAgents sets.String - deletedAgents sets.String + resultAgents sets.Set[string] + deletedAgents sets.Set[string] }{ "two new agents updated": { initAgents: []string{}, cacheAgents: "agent1,agent2", - resultAgents: sets.NewString(append([]string{"agent1", "agent2"}, util.DefaultCacheAgents...)...), - deletedAgents: sets.String{}, + resultAgents: sets.New(append([]string{"agent1", "agent2"}, util.DefaultCacheAgents...)...), + deletedAgents: sets.Set[string]{}, }, "two new agents updated but an old agent deleted": { initAgents: []string{"agent1", "agent2"}, cacheAgents: "agent2,agent3", - resultAgents: sets.NewString(append([]string{"agent2", "agent3"}, util.DefaultCacheAgents...)...), - deletedAgents: sets.NewString("agent1"), + resultAgents: sets.New(append([]string{"agent2", "agent3"}, util.DefaultCacheAgents...)...), + deletedAgents: sets.New("agent1"), }, "no agents updated ": { initAgents: []string{"agent1", "agent2"}, cacheAgents: "agent1,agent2", - resultAgents: sets.NewString(append([]string{"agent1", "agent2"}, util.DefaultCacheAgents...)...), - deletedAgents: sets.String{}, + resultAgents: sets.New(append([]string{"agent1", "agent2"}, util.DefaultCacheAgents...)...), + deletedAgents: sets.New[string](), }, "no agents updated with default": { initAgents: []string{"agent1", "agent2", "kubelet"}, cacheAgents: "agent1,agent2", - resultAgents: sets.NewString(append([]string{"agent1", "agent2"}, util.DefaultCacheAgents...)...), - deletedAgents: sets.String{}, + resultAgents: sets.New(append([]string{"agent1", "agent2"}, util.DefaultCacheAgents...)...), + deletedAgents: sets.New[string](), }, "empty agents added ": { initAgents: []string{}, cacheAgents: "", - resultAgents: sets.NewString(util.DefaultCacheAgents...), - deletedAgents: sets.String{}, + resultAgents: sets.New(util.DefaultCacheAgents...), + deletedAgents: sets.New[string](), }, } for k, tt := range testcases { t.Run(k, func(t *testing.T) { m := &CacheAgent{ - agents: sets.NewString(tt.initAgents...), + agents: sets.New(tt.initAgents...), } m.updateCacheAgents(strings.Join(tt.initAgents, ","), "") diff --git a/pkg/yurthub/cachemanager/cache_manager.go b/pkg/yurthub/cachemanager/cache_manager.go index ac16db535ab..a3464803d89 100644 --- a/pkg/yurthub/cachemanager/cache_manager.go +++ b/pkg/yurthub/cachemanager/cache_manager.go @@ -23,7 +23,6 @@ import ( "fmt" "io" "net/http" - "path" "path/filepath" "strconv" "strings" @@ -38,7 +37,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/watch" - "k8s.io/apiserver/pkg/endpoints/handlers" apirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/scheme" @@ -179,10 +177,6 @@ func (cm *cacheManager) queryListObject(req *http.Request) (runtime.Object, erro klog.Errorf("could not create ListObj for gvk %s for req: %s, %v", listGvk.String(), util.ReqString(req), err) return nil, err } - if err := setListObjSelfLink(listObj, req); err != nil { - klog.Errorf("could not set selfLink for ListObj of gvk %s for req: %s, %v", listGvk.String(), util.ReqString(req), err) - return nil, err - } objs, err := cm.storage.List(key) if err == storage.ErrStorageNotFound && isListRequestWithNameFieldSelector(req) { @@ -340,33 +334,6 @@ func generateEmptyListObjOfGVK(listGvk schema.GroupVersionKind) (runtime.Object, return listObj, nil } -func setListObjSelfLink(listObj runtime.Object, req *http.Request) error { - ctx := req.Context() - info, _ := apirequest.RequestInfoFrom(ctx) - clusterScoped := true - if info.Namespace != "" { - clusterScoped = false - } - - prefix := "/" + path.Join(info.APIGroup, info.APIVersion) - namer := handlers.ContextBasedNaming{ - SelfLinker: runtime.SelfLinker(meta.NewAccessor()), - SelfLinkPathPrefix: path.Join(prefix, info.Resource) + "/", - SelfLinkPathSuffix: "", - ClusterScoped: clusterScoped, - } - - uri, err := namer.GenerateListLink(req) - if err != nil { - return err - } - if err := namer.SetSelfLink(listObj, uri); err != nil { - klog.Infof("Unable to set self link on object: %v", err) - } - - return nil -} - func (cm *cacheManager) saveWatchObject(ctx context.Context, info *apirequest.RequestInfo, r io.ReadCloser, stopCh <-chan struct{}) error { delObjCnt := 0 updateObjCnt := 0 diff --git a/pkg/yurthub/certificate/manager/manager_test.go b/pkg/yurthub/certificate/manager/manager_test.go index 385a0fbcb30..78f3f877637 100644 --- a/pkg/yurthub/certificate/manager/manager_test.go +++ b/pkg/yurthub/certificate/manager/manager_test.go @@ -17,6 +17,7 @@ limitations under the License. package manager import ( + "context" "fmt" "net/url" "os" @@ -99,7 +100,7 @@ func TestReady(t *testing.T) { } mgr.Start() - err = wait.PollImmediate(2*time.Second, 1*time.Minute, func() (done bool, err error) { + err = wait.PollUntilContextTimeout(context.Background(), 2*time.Second, 1*time.Minute, true, func(ctx context.Context) (done bool, err error) { if mgr.Ready() { return true, nil } @@ -129,7 +130,7 @@ func TestReady(t *testing.T) { return } newMgr.Start() - err = wait.PollImmediate(2*time.Second, 1*time.Minute, func() (done bool, err error) { + err = wait.PollUntilContextTimeout(context.Background(), 2*time.Second, 1*time.Minute, true, func(ctx context.Context) (done bool, err error) { if mgr.Ready() { return true, nil } diff --git a/pkg/yurthub/certificate/server/server.go b/pkg/yurthub/certificate/server/server.go index 7f973face9b..f6aed0a9796 100644 --- a/pkg/yurthub/certificate/server/server.go +++ b/pkg/yurthub/certificate/server/server.go @@ -58,7 +58,7 @@ func NewHubServerCertificateManager(client clientset.Interface, clientCertManage kubeClientFn := func(current *tls.Certificate) (clientset.Interface, error) { // waiting for the certificate is generated - _ = wait.PollInfinite(5*time.Second, func() (bool, error) { + _ = wait.PollUntilContextCancel(context.Background(), 5*time.Second, true, func(ctx context.Context) (bool, error) { // keep polling until the yurthub client certificate is signed if clientCertManager.GetAPIServerClientCert() != nil { return true, nil diff --git a/pkg/yurthub/filter/approver/approver.go b/pkg/yurthub/filter/approver/approver.go index 6c5b5f6aea2..237fe1ee5df 100644 --- a/pkg/yurthub/filter/approver/approver.go +++ b/pkg/yurthub/filter/approver/approver.go @@ -38,10 +38,10 @@ import ( type approver struct { sync.Mutex - reqKeyToNames map[string]sets.String + reqKeyToNames map[string]sets.Set[string] configMapSynced cache.InformerSynced - supportedResourceAndVerbsForFilter map[string]map[string]sets.String - defaultReqKeyToNames map[string]sets.String + supportedResourceAndVerbsForFilter map[string]map[string]sets.Set[string] + defaultReqKeyToNames map[string]sets.Set[string] stopCh chan struct{} } @@ -50,20 +50,20 @@ var ( defaultBlackListRequests = sets.NewString(reqKey(projectinfo.GetHubName(), "configmaps", "list"), reqKey(projectinfo.GetHubName(), "configmaps", "watch")) ) -func NewApprover(sharedFactory informers.SharedInformerFactory, filterSupportedResAndVerbs map[string]map[string]sets.String) filter.Approver { +func NewApprover(sharedFactory informers.SharedInformerFactory, filterSupportedResAndVerbs map[string]map[string]sets.Set[string]) filter.Approver { configMapInformer := sharedFactory.Core().V1().ConfigMaps().Informer() na := &approver{ - reqKeyToNames: make(map[string]sets.String), + reqKeyToNames: make(map[string]sets.Set[string]), configMapSynced: configMapInformer.HasSynced, supportedResourceAndVerbsForFilter: filterSupportedResAndVerbs, - defaultReqKeyToNames: make(map[string]sets.String), + defaultReqKeyToNames: make(map[string]sets.Set[string]), stopCh: make(chan struct{}), } for name, setting := range options.SupportedComponentsForFilter { for _, key := range na.parseRequestSetting(name, setting) { if _, ok := na.defaultReqKeyToNames[key]; !ok { - na.defaultReqKeyToNames[key] = sets.NewString() + na.defaultReqKeyToNames[key] = sets.New[string]() } na.defaultReqKeyToNames[key].Insert(name) } @@ -109,12 +109,12 @@ func (a *approver) addConfigMap(obj interface{}) { } // get reqKeyToNames of user request setting from configmap - reqKeyToNamesFromCM := make(map[string]sets.String) + reqKeyToNamesFromCM := make(map[string]sets.Set[string]) for key, setting := range cm.Data { if filterName, ok := a.hasFilterName(key); ok { for _, requestKey := range a.parseRequestSetting(filterName, setting) { if _, ok := reqKeyToNamesFromCM[requestKey]; !ok { - reqKeyToNamesFromCM[requestKey] = sets.NewString() + reqKeyToNamesFromCM[requestKey] = sets.New[string]() } reqKeyToNamesFromCM[requestKey].Insert(filterName) } @@ -145,12 +145,12 @@ func (a *approver) updateConfigMap(oldObj, newObj interface{}) { } // get reqKeyToNames of user request setting from new configmap - reqKeyToNamesFromCM := make(map[string]sets.String) + reqKeyToNamesFromCM := make(map[string]sets.Set[string]) for key, setting := range newCM.Data { if filterName, ok := a.hasFilterName(key); ok { for _, requestKey := range a.parseRequestSetting(filterName, setting) { if _, ok := reqKeyToNamesFromCM[requestKey]; !ok { - reqKeyToNamesFromCM[requestKey] = sets.NewString() + reqKeyToNamesFromCM[requestKey] = sets.New[string]() } reqKeyToNamesFromCM[requestKey].Insert(filterName) } @@ -168,11 +168,11 @@ func (a *approver) deleteConfigMap(obj interface{}) { } // update reqKeyToName by merging user setting - a.merge("delete", map[string]sets.String{}) + a.merge("delete", map[string]sets.Set[string]{}) } // merge is used to add specified setting into reqKeyToNames map. -func (a *approver) merge(action string, keyToNamesSetting map[string]sets.String) { +func (a *approver) merge(action string, keyToNamesSetting map[string]sets.Set[string]) { a.Lock() defer a.Unlock() // remove current user setting from reqKeyToNames and left default setting @@ -188,7 +188,7 @@ func (a *approver) merge(action string, keyToNamesSetting map[string]sets.String // merge new user setting for key, names := range keyToNamesSetting { if _, ok := a.reqKeyToNames[key]; !ok { - a.reqKeyToNames[key] = sets.NewString() + a.reqKeyToNames[key] = sets.New[string]() } a.reqKeyToNames[key].Insert(names.UnsortedList()...) } @@ -212,7 +212,7 @@ func (a *approver) parseRequestSetting(name, setting string) []string { for resource, verbSet := range resourceAndVerbs { comp = strings.TrimSpace(comp) resource = strings.TrimSpace(resource) - verbs := verbSet.List() + verbs := verbSet.UnsortedList() if len(comp) != 0 && len(resource) != 0 && len(verbs) != 0 { for i := range verbs { diff --git a/pkg/yurthub/filter/approver/approver_test.go b/pkg/yurthub/filter/approver/approver_test.go index eb15b5997bc..8af02345739 100644 --- a/pkg/yurthub/filter/approver/approver_test.go +++ b/pkg/yurthub/filter/approver/approver_test.go @@ -20,6 +20,7 @@ import ( "net/http" "net/http/httptest" "reflect" + "sort" "testing" v1 "k8s.io/api/core/v1" @@ -38,16 +39,16 @@ import ( "github.com/openyurtio/openyurt/pkg/yurthub/proxy/util" ) -var supportedResourceAndVerbsForFilter = map[string]map[string]sets.String{ +var supportedResourceAndVerbsForFilter = map[string]map[string]sets.Set[string]{ masterservice.FilterName: { - "services": sets.NewString("list", "watch"), + "services": sets.New[string]("list", "watch"), }, discardcloudservice.FilterName: { - "services": sets.NewString("list", "watch"), + "services": sets.New[string]("list", "watch"), }, servicetopology.FilterName: { - "endpoints": sets.NewString("list", "watch"), - "endpointslices": sets.NewString("list", "watch"), + "endpoints": sets.New[string]("list", "watch"), + "endpointslices": sets.New[string]("list", "watch"), }, } @@ -195,7 +196,7 @@ func TestAddConfigMap(t *testing.T) { testcases := []struct { desc string cm *v1.ConfigMap - resultReqKeyToNames map[string]sets.String + resultReqKeyToNames map[string]sets.Set[string] }{ { desc: "add a new filter setting", @@ -237,7 +238,7 @@ func TestAddConfigMap(t *testing.T) { if !reflect.DeepEqual(approver.reqKeyToNames, tt.resultReqKeyToNames) { t.Errorf("expect reqkeyToNames is %#+v, but got %#+v", tt.resultReqKeyToNames, approver.reqKeyToNames) } - approver.merge("cleanup", map[string]sets.String{}) + approver.merge("cleanup", map[string]sets.Set[string]{}) }) } } @@ -248,7 +249,7 @@ func TestUpdateConfigMap(t *testing.T) { desc string oldCM *v1.ConfigMap newCM *v1.ConfigMap - resultReqKeyToNames map[string]sets.String + resultReqKeyToNames map[string]sets.Set[string] }{ { desc: "add a new filter setting", @@ -311,7 +312,7 @@ func TestUpdateConfigMap(t *testing.T) { if !reflect.DeepEqual(approver.reqKeyToNames, tt.resultReqKeyToNames) { t.Errorf("expect reqkeyToName is %#+v, but got %#+v", tt.resultReqKeyToNames, approver.reqKeyToNames) } - approver.merge("cleanup", map[string]sets.String{}) + approver.merge("cleanup", map[string]sets.Set[string]{}) }) } } @@ -320,20 +321,20 @@ func TestMerge(t *testing.T) { approver := newApprover(supportedResourceAndVerbsForFilter) testcases := map[string]struct { action string - reqKeyToNamesFromCM map[string]sets.String - resultReqKeyToNames map[string]sets.String + reqKeyToNamesFromCM map[string]sets.Set[string] + resultReqKeyToNames map[string]sets.Set[string] }{ "init req key to name": { action: "init", - reqKeyToNamesFromCM: map[string]sets.String{}, + reqKeyToNamesFromCM: map[string]sets.Set[string]{}, resultReqKeyToNames: approver.defaultReqKeyToNames, }, "add some items of req key to name": { action: "add", - reqKeyToNamesFromCM: map[string]sets.String{ - "comp1/resources1/list": sets.NewString("filter1"), - "comp2/resources2/watch": sets.NewString("filter2"), - "comp3/resources3/watch": sets.NewString("filter1"), + reqKeyToNamesFromCM: map[string]sets.Set[string]{ + "comp1/resources1/list": sets.New[string]("filter1"), + "comp2/resources2/watch": sets.New[string]("filter2"), + "comp3/resources3/watch": sets.New[string]("filter1"), }, resultReqKeyToNames: mergeReqKeyMap(approver.defaultReqKeyToNames, map[string]string{ "comp1/resources1/list": "filter1", @@ -343,9 +344,9 @@ func TestMerge(t *testing.T) { }, "update and delete item of req key to name": { action: "update", - reqKeyToNamesFromCM: map[string]sets.String{ - "comp1/resources1/list": sets.NewString("filter1"), - "comp2/resources2/watch": sets.NewString("filter3"), + reqKeyToNamesFromCM: map[string]sets.Set[string]{ + "comp1/resources1/list": sets.New[string]("filter1"), + "comp2/resources2/watch": sets.New[string]("filter3"), }, resultReqKeyToNames: mergeReqKeyMap(approver.defaultReqKeyToNames, map[string]string{ "comp1/resources1/list": "filter1", @@ -354,9 +355,9 @@ func TestMerge(t *testing.T) { }, "update default setting of req key to name": { action: "update", - reqKeyToNamesFromCM: map[string]sets.String{ - "kubelet/services/list": sets.NewString("filter1"), - "comp2/resources2/watch": sets.NewString("filter3"), + reqKeyToNamesFromCM: map[string]sets.Set[string]{ + "kubelet/services/list": sets.New("filter1"), + "comp2/resources2/watch": sets.New("filter3"), }, resultReqKeyToNames: mergeReqKeyMap(approver.defaultReqKeyToNames, map[string]string{ "comp2/resources2/watch": "filter3", @@ -365,7 +366,7 @@ func TestMerge(t *testing.T) { }, "clear all user setting of req key to name": { action: "update", - reqKeyToNamesFromCM: map[string]sets.String{}, + reqKeyToNamesFromCM: map[string]sets.Set[string]{}, resultReqKeyToNames: approver.defaultReqKeyToNames, }, } @@ -413,6 +414,8 @@ func TestParseRequestSetting(t *testing.T) { for k, tt := range testcases { t.Run(k, func(t *testing.T) { keys := approver.parseRequestSetting(tt.filterName, tt.filterSetting) + sort.Strings(keys) + sort.Strings(tt.resultKeys) if !reflect.DeepEqual(keys, tt.resultKeys) { t.Errorf("expect request keys %#+v, but got %#+v", tt.resultKeys, keys) @@ -639,35 +642,35 @@ func TestReqKey(t *testing.T) { } } -func mergeReqKeyMap(base map[string]sets.String, m map[string]string) map[string]sets.String { - reqKeyToNames := make(map[string]sets.String) +func mergeReqKeyMap(base map[string]sets.Set[string], m map[string]string) map[string]sets.Set[string] { + reqKeyToNames := make(map[string]sets.Set[string]) for k, v := range base { - reqKeyToNames[k] = sets.NewString(v.UnsortedList()...) + reqKeyToNames[k] = sets.New[string](v.UnsortedList()...) } for k, v := range m { if _, ok := reqKeyToNames[k]; ok { reqKeyToNames[k].Insert(v) } else { - reqKeyToNames[k] = sets.NewString(v) + reqKeyToNames[k] = sets.New[string](v) } } return reqKeyToNames } -func newApprover(filterSupportedResAndVerbs map[string]map[string]sets.String) *approver { +func newApprover(filterSupportedResAndVerbs map[string]map[string]sets.Set[string]) *approver { na := &approver{ - reqKeyToNames: make(map[string]sets.String), + reqKeyToNames: make(map[string]sets.Set[string]), supportedResourceAndVerbsForFilter: filterSupportedResAndVerbs, stopCh: make(chan struct{}), } - defaultReqKeyToFilterNames := make(map[string]sets.String) + defaultReqKeyToFilterNames := make(map[string]sets.Set[string]) for name, setting := range options.SupportedComponentsForFilter { for _, key := range na.parseRequestSetting(name, setting) { if _, ok := defaultReqKeyToFilterNames[key]; !ok { - defaultReqKeyToFilterNames[key] = sets.NewString() + defaultReqKeyToFilterNames[key] = sets.New[string]() } defaultReqKeyToFilterNames[key].Insert(name) } diff --git a/pkg/yurthub/filter/base/base.go b/pkg/yurthub/filter/base/base.go index c2596aa51c3..5fa4b99dfb2 100644 --- a/pkg/yurthub/filter/base/base.go +++ b/pkg/yurthub/filter/base/base.go @@ -32,14 +32,14 @@ type Filters struct { sync.Mutex names []string registry map[string]Factory - disabledFilters sets.String + disabledFilters sets.Set[string] } func NewFilters(disabledFilters []string) *Filters { return &Filters{ names: make([]string, 0), registry: make(map[string]Factory), - disabledFilters: sets.NewString(disabledFilters...), + disabledFilters: sets.New(disabledFilters...), } } @@ -54,18 +54,18 @@ func (fs *Filters) NewFromFilters(initializer filter.Initializer) ([]filter.Obje ins, err := factory() if err != nil { - klog.Errorf("new filter %s failed, %v", name, err) + klog.Errorf("couldn't new filter %s, %v", name, err) return nil, err } if err = initializer.Initialize(ins); err != nil { - klog.Errorf("Filter %s initialize failed, %v", name, err) + klog.Errorf("couldn't initialize filter %s, %v", name, err) return nil, err } - klog.V(2).Infof("Filter %s initialize successfully", name) + klog.V(2).Infof("filter %s initialize successfully", name) filters = append(filters, ins) } else { - klog.V(2).Infof("Filter %s is disabled", name) + klog.V(2).Infof("filter %s is disabled", name) } } diff --git a/pkg/yurthub/filter/base/base_test.go b/pkg/yurthub/filter/base/base_test.go index 01d9e98a4dc..9eeb64bf925 100644 --- a/pkg/yurthub/filter/base/base_test.go +++ b/pkg/yurthub/filter/base/base_test.go @@ -42,8 +42,8 @@ func (noh *nopObjectHandler) Name() string { return noh.name } -func (noh *nopObjectHandler) SupportedResourceAndVerbs() map[string]sets.String { - return map[string]sets.String{} +func (noh *nopObjectHandler) SupportedResourceAndVerbs() map[string]sets.Set[string] { + return map[string]sets.Set[string]{} } func (noh *nopObjectHandler) Filter(obj runtime.Object, stopCh <-chan struct{}) runtime.Object { @@ -87,45 +87,45 @@ func TestNewFromFilters(t *testing.T) { inputFilters []string disabledFilters []string initializer filter.Initializer - generatedFilters sets.String + generatedFilters sets.Set[string] expectedErr bool }{ "disable master service filter": { inputFilters: allFilters, disabledFilters: []string{"masterservice"}, - generatedFilters: sets.NewString(allFilters...).Delete("masterservice"), + generatedFilters: sets.New(allFilters...).Delete("masterservice"), }, "disable service topology filter": { inputFilters: allFilters, disabledFilters: []string{"servicetopology"}, - generatedFilters: sets.NewString(allFilters...).Delete("servicetopology"), + generatedFilters: sets.New(allFilters...).Delete("servicetopology"), }, "disable discard cloud service filter": { inputFilters: allFilters, disabledFilters: []string{"discardcloudservice"}, - generatedFilters: sets.NewString(allFilters...).Delete("discardcloudservice"), + generatedFilters: sets.New(allFilters...).Delete("discardcloudservice"), }, "disable all filters": { inputFilters: allFilters, disabledFilters: []string{"*"}, - generatedFilters: sets.NewString(), + generatedFilters: sets.New[string](), }, "register duplicated filters": { inputFilters: append(allFilters, "servicetopology"), disabledFilters: []string{}, - generatedFilters: sets.NewString(allFilters...), + generatedFilters: sets.New(allFilters...), }, "a invalid filter": { inputFilters: append(allFilters, "invalidFilter"), disabledFilters: []string{}, - generatedFilters: sets.NewString(), + generatedFilters: sets.New[string](), expectedErr: true, }, "initialize error": { inputFilters: allFilters, disabledFilters: []string{}, initializer: &errInitializer{}, - generatedFilters: sets.NewString(), + generatedFilters: sets.New[string](), expectedErr: true, }, } @@ -158,7 +158,7 @@ func TestNewFromFilters(t *testing.T) { return } - gotRunners := sets.NewString() + gotRunners := sets.New[string]() for i := range runners { gotRunners.Insert(runners[i].Name()) } diff --git a/pkg/yurthub/filter/discardcloudservice/filter.go b/pkg/yurthub/filter/discardcloudservice/filter.go index fa6d3eb551d..ac41cea303d 100644 --- a/pkg/yurthub/filter/discardcloudservice/filter.go +++ b/pkg/yurthub/filter/discardcloudservice/filter.go @@ -62,9 +62,9 @@ func (sf *discardCloudServiceFilter) Name() string { return FilterName } -func (sf *discardCloudServiceFilter) SupportedResourceAndVerbs() map[string]sets.String { - return map[string]sets.String{ - "services": sets.NewString("list", "watch"), +func (sf *discardCloudServiceFilter) SupportedResourceAndVerbs() map[string]sets.Set[string] { + return map[string]sets.Set[string]{ + "services": sets.New("list", "watch"), } } diff --git a/pkg/yurthub/filter/discardcloudservice/filter_test.go b/pkg/yurthub/filter/discardcloudservice/filter_test.go index d3cbc249449..40959fd736b 100644 --- a/pkg/yurthub/filter/discardcloudservice/filter_test.go +++ b/pkg/yurthub/filter/discardcloudservice/filter_test.go @@ -56,7 +56,7 @@ func TestSupportedResourceAndVerbs(t *testing.T) { t.Errorf("expect resource is services, but got %s", resource) } - if !verbs.Equal(sets.NewString("list", "watch")) { + if !verbs.Equal(sets.New("list", "watch")) { t.Errorf("expect verbs are list/watch, but got %v", verbs.UnsortedList()) } } diff --git a/pkg/yurthub/filter/forwardkubesvctraffic/filter.go b/pkg/yurthub/filter/forwardkubesvctraffic/filter.go index 02e5176a5af..f63d9c5feff 100644 --- a/pkg/yurthub/filter/forwardkubesvctraffic/filter.go +++ b/pkg/yurthub/filter/forwardkubesvctraffic/filter.go @@ -60,9 +60,9 @@ func (fkst *forwardKubeSVCTrafficFilter) Name() string { return FilterName } -func (fkst *forwardKubeSVCTrafficFilter) SupportedResourceAndVerbs() map[string]sets.String { - return map[string]sets.String{ - "endpointslices": sets.NewString("list", "watch"), +func (fkst *forwardKubeSVCTrafficFilter) SupportedResourceAndVerbs() map[string]sets.Set[string] { + return map[string]sets.Set[string]{ + "endpointslices": sets.New("list", "watch"), } } diff --git a/pkg/yurthub/filter/forwardkubesvctraffic/filter_test.go b/pkg/yurthub/filter/forwardkubesvctraffic/filter_test.go index c658cfac73c..adc51b529ae 100644 --- a/pkg/yurthub/filter/forwardkubesvctraffic/filter_test.go +++ b/pkg/yurthub/filter/forwardkubesvctraffic/filter_test.go @@ -59,7 +59,7 @@ func TestSupportedResourceAndVerbs(t *testing.T) { t.Errorf("expect resource is endpointslices, but got %s", resource) } - if !verbs.Equal(sets.NewString("list", "watch")) { + if !verbs.Equal(sets.New("list", "watch")) { t.Errorf("expect verbs are list/watch, but got %v", verbs.UnsortedList()) } } diff --git a/pkg/yurthub/filter/inclusterconfig/filter.go b/pkg/yurthub/filter/inclusterconfig/filter.go index 594a6602cfc..8fb1f426b32 100644 --- a/pkg/yurthub/filter/inclusterconfig/filter.go +++ b/pkg/yurthub/filter/inclusterconfig/filter.go @@ -57,9 +57,9 @@ func (iccf *inClusterConfigFilter) Name() string { return FilterName } -func (iccf *inClusterConfigFilter) SupportedResourceAndVerbs() map[string]sets.String { - return map[string]sets.String{ - "configmaps": sets.NewString("get", "list", "watch"), +func (iccf *inClusterConfigFilter) SupportedResourceAndVerbs() map[string]sets.Set[string] { + return map[string]sets.Set[string]{ + "configmaps": sets.New("get", "list", "watch"), } } diff --git a/pkg/yurthub/filter/inclusterconfig/filter_test.go b/pkg/yurthub/filter/inclusterconfig/filter_test.go index 694d422f440..577bfd2ea97 100644 --- a/pkg/yurthub/filter/inclusterconfig/filter_test.go +++ b/pkg/yurthub/filter/inclusterconfig/filter_test.go @@ -56,7 +56,7 @@ func TestSupportedResourceAndVerbs(t *testing.T) { t.Errorf("expect resource is services, but got %s", resource) } - if !verbs.Equal(sets.NewString("get", "list", "watch")) { + if !verbs.Equal(sets.New("get", "list", "watch")) { t.Errorf("expect verbs are get/list/watch, but got %v", verbs.UnsortedList()) } } diff --git a/pkg/yurthub/filter/initializer/initializer_test.go b/pkg/yurthub/filter/initializer/initializer_test.go index dbaa2892ae1..4671605b391 100644 --- a/pkg/yurthub/filter/initializer/initializer_test.go +++ b/pkg/yurthub/filter/initializer/initializer_test.go @@ -130,8 +130,8 @@ func (bef *baseErrFilter) Name() string { return "nop" } -func (bef *baseErrFilter) SupportedResourceAndVerbs() map[string]sets.String { - return map[string]sets.String{} +func (bef *baseErrFilter) SupportedResourceAndVerbs() map[string]sets.Set[string] { + return map[string]sets.Set[string]{} } func (bef *baseErrFilter) Filter(obj runtime.Object, _ <-chan struct{}) runtime.Object { diff --git a/pkg/yurthub/filter/initializer/node_initializer_test.go b/pkg/yurthub/filter/initializer/node_initializer_test.go index 95acb34cda8..284b3ee5a7d 100644 --- a/pkg/yurthub/filter/initializer/node_initializer_test.go +++ b/pkg/yurthub/filter/initializer/node_initializer_test.go @@ -45,8 +45,8 @@ func (nop *nopNodeHandler) Name() string { return nop.name } -func (nop *nopNodeHandler) SupportedResourceAndVerbs() map[string]sets.String { - return map[string]sets.String{} +func (nop *nopNodeHandler) SupportedResourceAndVerbs() map[string]sets.Set[string] { + return map[string]sets.Set[string]{} } func (nop *nopNodeHandler) Filter(obj runtime.Object, stopCh <-chan struct{}) runtime.Object { @@ -75,7 +75,7 @@ func TestNodesInitializer(t *testing.T) { enablePoolServiceTopology bool poolName string yurtClient *fake.FakeDynamicClient - expectedNodes sets.String + expectedNodes sets.Set[string] expectedErr bool }{ "get nodes in nodepool": { @@ -99,7 +99,7 @@ func TestNodesInitializer(t *testing.T) { }, }, ), - expectedNodes: sets.NewString("node1", "node2", "node3"), + expectedNodes: sets.New("node1", "node2", "node3"), }, "get nodes in nodebucket": { enableNodePool: true, @@ -123,7 +123,7 @@ func TestNodesInitializer(t *testing.T) { }, }, ), - expectedNodes: sets.NewString("node1", "node2"), + expectedNodes: sets.New("node1", "node2"), }, "nodepool doesn't exist": { enableNodePool: true, @@ -145,7 +145,7 @@ func TestNodesInitializer(t *testing.T) { poolName: "hangzhou", yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, nodeBucketGVRToListKind), expectedErr: false, - expectedNodes: sets.NewString(), + expectedNodes: sets.New[string](), }, } @@ -181,7 +181,7 @@ func TestNodesInitializer(t *testing.T) { return } - if !tc.expectedNodes.Equal(sets.NewString(nodes...)) { + if !tc.expectedNodes.Equal(sets.New(nodes...)) { t.Errorf("expect nodes %v, but got %v", tc.expectedNodes, nodes) } }) diff --git a/pkg/yurthub/filter/interfaces.go b/pkg/yurthub/filter/interfaces.go index 0e10ef943e7..ca1ca83528c 100644 --- a/pkg/yurthub/filter/interfaces.go +++ b/pkg/yurthub/filter/interfaces.go @@ -53,7 +53,7 @@ type ObjectFilter interface { Name() string // SupportedResourceAndVerbs is used to specify which resource and request verb is supported by the filter. // Because each filter can make sure what requests with resource and verb can be handled. - SupportedResourceAndVerbs() map[string]sets.String + SupportedResourceAndVerbs() map[string]sets.Set[string] // Filter is used for filtering runtime object // all filter logic should be located in it. Filter(obj runtime.Object, stopCh <-chan struct{}) runtime.Object diff --git a/pkg/yurthub/filter/manager/manager.go b/pkg/yurthub/filter/manager/manager.go index 5858310eafb..c5bd712e641 100644 --- a/pkg/yurthub/filter/manager/manager.go +++ b/pkg/yurthub/filter/manager/manager.go @@ -82,7 +82,7 @@ func NewFilterManager(options *yurtoptions.YurtHubOptions, serializerManager: serializerManager, } - filterSupportedResAndVerbs := make(map[string]map[string]sets.String) + filterSupportedResAndVerbs := make(map[string]map[string]sets.Set[string]) for i := range objFilters { m.nameToObjectFilter[objFilters[i].Name()] = objFilters[i] filterSupportedResAndVerbs[objFilters[i].Name()] = objFilters[i].SupportedResourceAndVerbs() diff --git a/pkg/yurthub/filter/manager/manager_test.go b/pkg/yurthub/filter/manager/manager_test.go index abfbbf74f20..27998c97df9 100644 --- a/pkg/yurthub/filter/manager/manager_test.go +++ b/pkg/yurthub/filter/manager/manager_test.go @@ -56,7 +56,7 @@ func TestFindResponseFilter(t *testing.T) { path string mgrIsNil bool isFound bool - names sets.String + names sets.Set[string] }{ "disable resource filter": { enableResourceFilter: false, @@ -69,7 +69,7 @@ func TestFindResponseFilter(t *testing.T) { verb: "GET", path: "/api/v1/services", isFound: true, - names: sets.NewString("masterservice"), + names: sets.New("masterservice"), }, "get discard cloud service and node port isolation filter": { enableResourceFilter: true, @@ -78,7 +78,7 @@ func TestFindResponseFilter(t *testing.T) { verb: "GET", path: "/api/v1/services", isFound: true, - names: sets.NewString("discardcloudservice", "nodeportisolation"), + names: sets.New("discardcloudservice", "nodeportisolation"), }, "get service topology filter": { enableResourceFilter: true, @@ -87,7 +87,7 @@ func TestFindResponseFilter(t *testing.T) { verb: "GET", path: "/api/v1/endpoints", isFound: true, - names: sets.NewString("servicetopology"), + names: sets.New("servicetopology"), }, "disable service topology filter": { enableResourceFilter: true, @@ -105,7 +105,7 @@ func TestFindResponseFilter(t *testing.T) { verb: "GET", path: "/api/v1/services", isFound: true, - names: sets.NewString("nodeportisolation"), + names: sets.New("nodeportisolation"), }, } @@ -163,8 +163,8 @@ func TestFindResponseFilter(t *testing.T) { } names := strings.Split(responseFilter.Name(), ",") - if !tt.names.Equal(sets.NewString(names...)) { - t.Errorf("expect filter names %v, but got %v", tt.names.List(), names) + if !tt.names.Equal(sets.New(names...)) { + t.Errorf("expect filter names %v, but got %v", sets.List(tt.names), names) } }) } diff --git a/pkg/yurthub/filter/masterservice/filter.go b/pkg/yurthub/filter/masterservice/filter.go index bd47d1cb19c..eab0450f143 100644 --- a/pkg/yurthub/filter/masterservice/filter.go +++ b/pkg/yurthub/filter/masterservice/filter.go @@ -58,9 +58,9 @@ func (msf *masterServiceFilter) Name() string { return FilterName } -func (msf *masterServiceFilter) SupportedResourceAndVerbs() map[string]sets.String { - return map[string]sets.String{ - "services": sets.NewString("list", "watch"), +func (msf *masterServiceFilter) SupportedResourceAndVerbs() map[string]sets.Set[string] { + return map[string]sets.Set[string]{ + "services": sets.New("list", "watch"), } } diff --git a/pkg/yurthub/filter/masterservice/filter_test.go b/pkg/yurthub/filter/masterservice/filter_test.go index e1e390a4368..810d5210428 100644 --- a/pkg/yurthub/filter/masterservice/filter_test.go +++ b/pkg/yurthub/filter/masterservice/filter_test.go @@ -56,7 +56,7 @@ func TestSupportedResourceAndVerbs(t *testing.T) { t.Errorf("expect resource is services, but got %s", resource) } - if !verbs.Equal(sets.NewString("list", "watch")) { + if !verbs.Equal(sets.New("list", "watch")) { t.Errorf("expect verbs are list/watch, but got %v", verbs.UnsortedList()) } } diff --git a/pkg/yurthub/filter/nodeportisolation/filter.go b/pkg/yurthub/filter/nodeportisolation/filter.go index e6f0a602b08..21aee4633e0 100644 --- a/pkg/yurthub/filter/nodeportisolation/filter.go +++ b/pkg/yurthub/filter/nodeportisolation/filter.go @@ -62,9 +62,9 @@ func (nif *nodePortIsolationFilter) Name() string { return FilterName } -func (nif *nodePortIsolationFilter) SupportedResourceAndVerbs() map[string]sets.String { - return map[string]sets.String{ - "services": sets.NewString("list", "watch"), +func (nif *nodePortIsolationFilter) SupportedResourceAndVerbs() map[string]sets.Set[string] { + return map[string]sets.Set[string]{ + "services": sets.New("list", "watch"), } } @@ -128,8 +128,8 @@ func (nif *nodePortIsolationFilter) resolveNodePoolName() string { return nif.nodePoolName } -func getNodePoolConfiguration(v string) sets.String { - nodePoolConf := sets.NewString() +func getNodePoolConfiguration(v string) sets.Set[string] { + nodePoolConf := sets.New[string]() nodePoolsForValidation := sets.NewString() for _, name := range strings.Split(v, ",") { name = strings.TrimSpace(name) @@ -143,7 +143,7 @@ func getNodePoolConfiguration(v string) sets.String { return nodePoolConf } -func isNodePoolEnabled(nodePoolConf sets.String, name string) bool { +func isNodePoolEnabled(nodePoolConf sets.Set[string], name string) bool { if nodePoolConf.Has(name) { return true } diff --git a/pkg/yurthub/filter/nodeportisolation/filter_test.go b/pkg/yurthub/filter/nodeportisolation/filter_test.go index d90ca675344..43d87f941bc 100644 --- a/pkg/yurthub/filter/nodeportisolation/filter_test.go +++ b/pkg/yurthub/filter/nodeportisolation/filter_test.go @@ -58,7 +58,7 @@ func TestSupportedResourceAndVerbs(t *testing.T) { t.Errorf("expect resource is services, but got %s", resource) } - if !verbs.Equal(sets.NewString("list", "watch")) { + if !verbs.Equal(sets.New("list", "watch")) { t.Errorf("expect verbs are list/watch, but got %v", verbs.UnsortedList()) } } diff --git a/pkg/yurthub/filter/responsefilter/filter.go b/pkg/yurthub/filter/responsefilter/filter.go index afcc60bbd0d..eb6df24994c 100644 --- a/pkg/yurthub/filter/responsefilter/filter.go +++ b/pkg/yurthub/filter/responsefilter/filter.go @@ -230,9 +230,9 @@ func (chain filterChain) Name() string { return strings.Join(names, ",") } -func (chain filterChain) SupportedResourceAndVerbs() map[string]sets.String { +func (chain filterChain) SupportedResourceAndVerbs() map[string]sets.Set[string] { // do nothing - return map[string]sets.String{} + return map[string]sets.Set[string]{} } func (chain filterChain) Filter(obj runtime.Object, stopCh <-chan struct{}) runtime.Object { diff --git a/pkg/yurthub/filter/responsefilter/filter_test.go b/pkg/yurthub/filter/responsefilter/filter_test.go index c6595345d68..dfdddd2904a 100644 --- a/pkg/yurthub/filter/responsefilter/filter_test.go +++ b/pkg/yurthub/filter/responsefilter/filter_test.go @@ -73,8 +73,8 @@ func (noh *nopObjectHandler) Name() string { return noh.name } -func (noh *nopObjectHandler) SupportedResourceAndVerbs() map[string]sets.String { - return map[string]sets.String{} +func (noh *nopObjectHandler) SupportedResourceAndVerbs() map[string]sets.Set[string] { + return map[string]sets.Set[string]{} } func (noh *nopObjectHandler) Filter(obj runtime.Object, stopCh <-chan struct{}) runtime.Object { @@ -2340,7 +2340,7 @@ func TestResponseFilterForWatchRequest(t *testing.T) { accept string eventType watch.EventType inputObj runtime.Object - names sets.String + names sets.Set[string] expectedObj runtime.Object expectedEventType watch.EventType }{ @@ -2367,7 +2367,7 @@ func TestResponseFilterForWatchRequest(t *testing.T) { Type: corev1.ServiceTypeLoadBalancer, }, }, - names: sets.NewString("discardcloudservice"), + names: sets.New("discardcloudservice"), expectedObj: &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -2410,7 +2410,7 @@ func TestResponseFilterForWatchRequest(t *testing.T) { Type: corev1.ServiceTypeLoadBalancer, }, }, - names: sets.NewString("discardcloudservice", "nodeportisolation"), + names: sets.New("discardcloudservice", "nodeportisolation"), expectedObj: &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -2450,7 +2450,7 @@ func TestResponseFilterForWatchRequest(t *testing.T) { Type: corev1.ServiceTypeLoadBalancer, }, }, - names: sets.NewString("discardcloudservice", "nodeportisolation"), + names: sets.New("discardcloudservice", "nodeportisolation"), expectedObj: &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -2516,8 +2516,8 @@ func TestResponseFilterForWatchRequest(t *testing.T) { } names := strings.Split(responseFilter.Name(), ",") - if !tc.names.Equal(sets.NewString(names...)) { - t.Errorf("expect filter names %v, but got %v", tc.names.List(), names) + if !tc.names.Equal(sets.New(names...)) { + t.Errorf("expect filter names %v, but got %v", sets.List(tc.names), names) return } diff --git a/pkg/yurthub/filter/servicetopology/filter.go b/pkg/yurthub/filter/servicetopology/filter.go index 2cbd38f969a..04d91fb4743 100644 --- a/pkg/yurthub/filter/servicetopology/filter.go +++ b/pkg/yurthub/filter/servicetopology/filter.go @@ -73,10 +73,10 @@ func (stf *serviceTopologyFilter) Name() string { return FilterName } -func (stf *serviceTopologyFilter) SupportedResourceAndVerbs() map[string]sets.String { - return map[string]sets.String{ - "endpoints": sets.NewString("list", "watch"), - "endpointslices": sets.NewString("list", "watch"), +func (stf *serviceTopologyFilter) SupportedResourceAndVerbs() map[string]sets.Set[string] { + return map[string]sets.Set[string]{ + "endpoints": sets.New("list", "watch"), + "endpointslices": sets.New("list", "watch"), } } diff --git a/pkg/yurthub/filter/servicetopology/filter_test.go b/pkg/yurthub/filter/servicetopology/filter_test.go index ba945d54db4..956512b7f3d 100644 --- a/pkg/yurthub/filter/servicetopology/filter_test.go +++ b/pkg/yurthub/filter/servicetopology/filter_test.go @@ -73,7 +73,7 @@ func TestSupportedResourceAndVerbs(t *testing.T) { t.Errorf("expect resource is endpoints/endpointslices, but got %s", resource) } - if !verbs.Equal(sets.NewString("list", "watch")) { + if !verbs.Equal(sets.New("list", "watch")) { t.Errorf("expect verbs are list/watch, but got %v", verbs.UnsortedList()) } } diff --git a/pkg/yurthub/healthchecker/node_lease.go b/pkg/yurthub/healthchecker/node_lease.go index cd48ba0b222..a00d37fbbba 100644 --- a/pkg/yurthub/healthchecker/node_lease.go +++ b/pkg/yurthub/healthchecker/node_lease.go @@ -142,8 +142,8 @@ func (nl *nodeLeaseImpl) newLease(base *coordinationv1.Lease) *coordinationv1.Le Namespace: corev1.NamespaceNodeLease, }, Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.StringPtr(nl.holderIdentity), - LeaseDurationSeconds: pointer.Int32Ptr(nl.leaseDurationSeconds), + HolderIdentity: pointer.String(nl.holderIdentity), + LeaseDurationSeconds: pointer.Int32(nl.leaseDurationSeconds), }, } } else { diff --git a/pkg/yurthub/healthchecker/node_lease_test.go b/pkg/yurthub/healthchecker/node_lease_test.go index e1776b242c9..4bb89034460 100644 --- a/pkg/yurthub/healthchecker/node_lease_test.go +++ b/pkg/yurthub/healthchecker/node_lease_test.go @@ -70,7 +70,7 @@ func TestNodeLeaseManager_Update(t *testing.T) { success: true, }, { - desc: "update fail", + desc: "couldn't update", updateReactor: func(action clienttesting.Action) (bool, runtime.Object, error) { return true, nil, noConnectionUpdateErr }, diff --git a/pkg/yurthub/kubernetes/rest/config_test.go b/pkg/yurthub/kubernetes/rest/config_test.go index bf76a17da04..d7c8e25673c 100644 --- a/pkg/yurthub/kubernetes/rest/config_test.go +++ b/pkg/yurthub/kubernetes/rest/config_test.go @@ -17,6 +17,7 @@ limitations under the License. package rest import ( + "context" "net/url" "os" "testing" @@ -61,7 +62,7 @@ func TestGetRestConfig(t *testing.T) { defer certManager.Stop() defer os.RemoveAll(testDir) - err = wait.PollImmediate(2*time.Second, 1*time.Minute, func() (done bool, err error) { + err = wait.PollUntilContextTimeout(context.Background(), 2*time.Second, 1*time.Minute, true, func(ctx context.Context) (done bool, err error) { if certManager.Ready() { return true, nil } diff --git a/pkg/yurthub/network/iptables.go b/pkg/yurthub/network/iptables.go index 07a84a6f70b..2657976b4f8 100644 --- a/pkg/yurthub/network/iptables.go +++ b/pkg/yurthub/network/iptables.go @@ -40,9 +40,9 @@ type IptablesManager struct { } func NewIptablesManager(dummyIfIP, dummyIfPort string) *IptablesManager { - protocol := iptables.ProtocolIpv4 + protocol := iptables.ProtocolIPv4 if utilnet.IsIPv6String(dummyIfIP) { - protocol = iptables.ProtocolIpv6 + protocol = iptables.ProtocolIPv6 } execer := exec.New() iptInterface := iptables.New(execer, protocol) diff --git a/pkg/yurthub/otaupdate/ota.go b/pkg/yurthub/otaupdate/ota.go index 48cdcacfc9e..4c551dfd159 100644 --- a/pkg/yurthub/otaupdate/ota.go +++ b/pkg/yurthub/otaupdate/ota.go @@ -148,7 +148,7 @@ func UpdatePod(clientset kubernetes.Interface, nodeName string) http.Handler { func preCheck(clientset kubernetes.Interface, namespace, podName, nodeName string) (*corev1.Pod, bool) { pod, err := clientset.CoreV1().Pods(namespace).Get(context.TODO(), podName, metav1.GetOptions{}) if err != nil { - klog.Errorf("Get pod %v/%v failed, %v", namespace, podName, err) + klog.Errorf("couldn't get pod %s/%s, %v", namespace, podName, err) return nil, false } diff --git a/pkg/yurthub/otaupdate/ota_test.go b/pkg/yurthub/otaupdate/ota_test.go index e15224ad18a..61d7a7ae337 100644 --- a/pkg/yurthub/otaupdate/ota_test.go +++ b/pkg/yurthub/otaupdate/ota_test.go @@ -39,7 +39,7 @@ func TestGetPods(t *testing.T) { dir := t.TempDir() dStorage, err := disk.NewDiskStorage(dir) if err != nil { - t.Errorf("failed to create disk storage, %v", err) + t.Errorf("couldn't to create disk storage, %v", err) } sWrapper := cachemanager.NewStorageWrapper(dStorage) @@ -58,11 +58,11 @@ func TestGetPods(t *testing.T) { Name: pod.Name, }) if err != nil { - t.Errorf("failed to get key, %v", err) + t.Errorf("couldn't to get key, %v", err) } err = sWrapper.Create(key, pod) if err != nil { - t.Errorf("failed to create obj, %v", err) + t.Errorf("cloudn't to create obj, %v", err) } } @@ -126,7 +126,7 @@ func Test_preCheck(t *testing.T) { assert.Equal(t, true, ok) }) - t.Run("Test_preCheckOKGetPodFailed", func(t *testing.T) { + t.Run("Test_preCheckCanNotGetPod", func(t *testing.T) { _, ok := preCheck(clientset, metav1.NamespaceDefault, "nginx1", "node") assert.Equal(t, false, ok) }) diff --git a/pkg/yurthub/otaupdate/upgrader/daemon_pod.go b/pkg/yurthub/otaupdate/upgrader/daemon_pod.go index 15b17ca74f9..ad7623d748e 100644 --- a/pkg/yurthub/otaupdate/upgrader/daemon_pod.go +++ b/pkg/yurthub/otaupdate/upgrader/daemon_pod.go @@ -34,10 +34,10 @@ type DaemonPodUpgrader struct { func (s *DaemonPodUpgrader) Apply() error { err := s.CoreV1().Pods(s.Namespace).Delete(context.TODO(), s.Name, metav1.DeleteOptions{}) if err != nil { - klog.Errorf("Update pod %v/%v failed when delete, %v", s.Namespace, s.Name, err) + klog.Errorf("couldn't update pod %s/%s because of can't delete, %v", s.Namespace, s.Name, err) return err } - klog.Infof("Start updating pod: %v/%v", s.Namespace, s.Name) + klog.Infof("Start updating pod: %s/%s", s.Namespace, s.Name) return nil } diff --git a/pkg/yurthub/proxy/util/util.go b/pkg/yurthub/proxy/util/util.go index ab2ae0d3e7a..7698fb0a21b 100644 --- a/pkg/yurthub/proxy/util/util.go +++ b/pkg/yurthub/proxy/util/util.go @@ -424,7 +424,7 @@ func WriteObject(statusCode int, obj runtime.Object, w http.ResponseWriter, req Version: info.APIVersion, } negotiatedSerializer := serializer.YurtHubSerializer.GetNegotiatedSerializer(gv.WithResource(info.Resource)) - responsewriters.WriteObjectNegotiated(negotiatedSerializer, negotiation.DefaultEndpointRestrictions, gv, w, req, statusCode, obj) + responsewriters.WriteObjectNegotiated(negotiatedSerializer, negotiation.DefaultEndpointRestrictions, gv, w, req, statusCode, obj, false) return nil } diff --git a/pkg/yurthub/server/certificate_test.go b/pkg/yurthub/server/certificate_test.go index 2451f1b057a..6b483b2bd4c 100644 --- a/pkg/yurthub/server/certificate_test.go +++ b/pkg/yurthub/server/certificate_test.go @@ -18,6 +18,7 @@ package server import ( "bytes" + "context" "encoding/json" "net/http" "net/http/httptest" @@ -59,7 +60,7 @@ func TestUpdateTokenHandler(t *testing.T) { defer certManager.Stop() defer os.RemoveAll(testDir) - err = wait.PollImmediate(2*time.Second, 1*time.Minute, func() (done bool, err error) { + err = wait.PollUntilContextTimeout(context.Background(), 2*time.Second, 1*time.Minute, true, func(ctx context.Context) (done bool, err error) { if certManager.GetAPIServerClientCert() != nil { return true, nil } @@ -73,7 +74,7 @@ func TestUpdateTokenHandler(t *testing.T) { body map[string]string statusCode int }{ - "failed to update join token": { + "can not update join token": { body: map[string]string{}, statusCode: http.StatusBadRequest, }, diff --git a/pkg/yurthub/server/nonresource.go b/pkg/yurthub/server/nonresource.go index 417c1b8c4af..96f110a2583 100644 --- a/pkg/yurthub/server/nonresource.go +++ b/pkg/yurthub/server/nonresource.go @@ -95,7 +95,7 @@ func nonResourceHandler(kubeClient *kubernetes.Clientset, sw cachemanager.Storag } result := kubeClient.RESTClient().Get().AbsPath(path).Do(context.TODO()) - code := pointer.IntPtr(0) + code := pointer.Int(0) result.StatusCode(code) if result.Error() != nil { err := result.Error() diff --git a/pkg/yurthub/server/nonresource_test.go b/pkg/yurthub/server/nonresource_test.go index 759ecae9c77..2b6e50466f4 100644 --- a/pkg/yurthub/server/nonresource_test.go +++ b/pkg/yurthub/server/nonresource_test.go @@ -70,7 +70,7 @@ func TestLocalCacheHandler(t *testing.T) { statusCode int metav1StatusCode int }{ - "failed to get from local cache, because it does not exist": { + "can not get from local cache, because it does not exist": { path: "/version", initData: nil, statusCode: http.StatusNotFound, @@ -149,13 +149,13 @@ func TestNonResourceHandler(t *testing.T) { initData: []byte("fake resource"), statusCode: http.StatusOK, }, - "failed to get non resource because of internal error": { + "can not get non resource because of internal error": { path: "/apis/discovery.k8s.io/v1beta1", err: internalError, statusCode: http.StatusInternalServerError, metav1StatusCode: http.StatusInternalServerError, }, - "failed to get non resource because it does not exist": { + "can not get non resource because it does not exist": { path: "/apis/discovery.k8s.io/v1", err: notFoundError, statusCode: http.StatusNotFound, diff --git a/pkg/yurthub/server/server.go b/pkg/yurthub/server/server.go index 1dff97ddf14..194ca1ca896 100644 --- a/pkg/yurthub/server/server.go +++ b/pkg/yurthub/server/server.go @@ -68,7 +68,7 @@ func RunYurtHubServers(cfg *config.YurtHubConfiguration, } if cfg.YurtHubSecureProxyServerServing != nil { - if _, err := cfg.YurtHubSecureProxyServerServing.Serve(proxyHandler, 0, stopCh); err != nil { + if _, _, err := cfg.YurtHubSecureProxyServerServing.Serve(proxyHandler, 0, stopCh); err != nil { return err } } diff --git a/pkg/yurthub/util/util.go b/pkg/yurthub/util/util.go index 65e63c547c2..0234c804f64 100644 --- a/pkg/yurthub/util/util.go +++ b/pkg/yurthub/util/util.go @@ -187,7 +187,7 @@ func WriteObject(statusCode int, obj runtime.Object, w http.ResponseWriter, req Version: info.APIVersion, } negotiatedSerializer := serializer.YurtHubSerializer.GetNegotiatedSerializer(gv.WithResource(info.Resource)) - responsewriters.WriteObjectNegotiated(negotiatedSerializer, negotiation.DefaultEndpointRestrictions, gv, w, req, statusCode, obj) + responsewriters.WriteObjectNegotiated(negotiatedSerializer, negotiation.DefaultEndpointRestrictions, gv, w, req, statusCode, obj, false) return nil } diff --git a/pkg/yurthub/yurtcoordinator/certmanager/certmanager_test.go b/pkg/yurthub/yurtcoordinator/certmanager/certmanager_test.go index a24aaf89381..53c23e6840d 100644 --- a/pkg/yurthub/yurtcoordinator/certmanager/certmanager_test.go +++ b/pkg/yurthub/yurtcoordinator/certmanager/certmanager_test.go @@ -17,6 +17,7 @@ limitations under the License. package certmanager import ( + "context" "fmt" "path/filepath" "reflect" @@ -236,7 +237,7 @@ func TestSecretAdd(t *testing.T) { // Expect to timeout which indicates the CertManager does not save the cert // that is not yurt-coordinator-yurthub-certs. - err = wait.PollImmediate(50*time.Millisecond, 10*time.Second, func() (done bool, err error) { + err = wait.PollUntilContextTimeout(context.Background(), 50*time.Millisecond, 10*time.Second, true, func(ctx context.Context) (done bool, err error) { if certMgr.secret != nil { return false, fmt.Errorf("unexpect cert initialization") } @@ -250,7 +251,7 @@ func TestSecretAdd(t *testing.T) { return false, nil }) - if err != wait.ErrWaitTimeout { + if !wait.Interrupted(err) { t.Errorf("CertManager should not react for add event of secret that is not yurt-coordinator-yurthub-certs, %v", err) } @@ -270,7 +271,7 @@ func TestSecretAdd(t *testing.T) { t.Errorf("failed to add secret %s, %v", yurtCoordinatorSecret.Name, err) } - err = wait.PollImmediate(50*time.Millisecond, 10*time.Second, func() (done bool, err error) { + err = wait.PollUntilContextTimeout(context.Background(), 50*time.Millisecond, 10*time.Second, true, func(ctx context.Context) (done bool, err error) { if pass, err := checkSecret(certMgr, yurtCoordinatorSecret, []expectFile{ { FilePath: certMgr.GetFilePath(RootCA), @@ -328,7 +329,7 @@ func TestSecretUpdate(t *testing.T) { t.Errorf("failed to add secret %s, %v", yurtCoordinatorSecret.Name, err) } - err = wait.Poll(50*time.Millisecond, 10*time.Second, func() (done bool, err error) { + err = wait.PollUntilContextTimeout(context.Background(), 50*time.Millisecond, 10*time.Second, true, func(ctx context.Context) (done bool, err error) { if pass, err := checkSecret(certMgr, yurtCoordinatorSecret, []expectFile{ { FilePath: certMgr.GetFilePath(RootCA), @@ -375,7 +376,7 @@ func TestSecretUpdate(t *testing.T) { t.Errorf("failed to update secret, %v", err) } - err = wait.PollImmediate(50*time.Millisecond, 10*time.Second, func() (done bool, err error) { + err = wait.PollUntilContextTimeout(context.Background(), 50*time.Millisecond, 10*time.Second, true, func(ctx context.Context) (done bool, err error) { if pass, err := checkSecret(certMgr, newSecret, []expectFile{ { FilePath: certMgr.GetFilePath(RootCA), @@ -421,7 +422,7 @@ func TestSecretUpdate(t *testing.T) { t.Errorf("failed to update secret, %v", err) } - err = wait.PollImmediate(50*time.Millisecond, 10*time.Second, func() (done bool, err error) { + err = wait.PollUntilContextTimeout(context.Background(), 50*time.Millisecond, 10*time.Second, true, func(ctx context.Context) (done bool, err error) { if pass, err := checkSecret(certMgr, newSecret, []expectFile{ { FilePath: certMgr.GetFilePath(RootCA), @@ -483,7 +484,7 @@ func TestSecretDelete(t *testing.T) { t.Errorf("failed to add secret %s, %v", yurtCoordinatorSecret.Name, err) } - err = wait.PollImmediate(50*time.Millisecond, 10*time.Second, func() (done bool, err error) { + err = wait.PollUntilContextTimeout(context.Background(), 50*time.Millisecond, 10*time.Second, true, func(ctx context.Context) (done bool, err error) { if pass, err := checkSecret(certMgr, yurtCoordinatorSecret, []expectFile{ { FilePath: certMgr.GetFilePath(RootCA), @@ -526,7 +527,7 @@ func TestSecretDelete(t *testing.T) { t.Errorf("failed to delete secret, %v", err) } - err = wait.PollImmediate(50*time.Millisecond, 10*time.Second, func() (done bool, err error) { + err = wait.PollUntilContextTimeout(context.Background(), 50*time.Millisecond, 10*time.Second, true, func(ctx context.Context) (done bool, err error) { if certMgr.GetAPIServerClientCert() != nil { return false, nil } diff --git a/pkg/yurthub/yurtcoordinator/coordinator_test.go b/pkg/yurthub/yurtcoordinator/coordinator_test.go index eb284ed7e26..a7f0453326d 100644 --- a/pkg/yurthub/yurtcoordinator/coordinator_test.go +++ b/pkg/yurthub/yurtcoordinator/coordinator_test.go @@ -168,12 +168,12 @@ func TestInformerSyncLeaseDelete(t *testing.T) { poolCacheSyncedDetector.EnsureStart() defer poolCacheSyncedDetector.EnsureStop() - err := wait.PollUntil(50*time.Millisecond, func() (done bool, err error) { + err := wait.PollUntilContextCancel(ctx, 50*time.Millisecond, true, func(ctx2 context.Context) (done bool, err error) { if isPoolCacheSynced { return true, nil } return false, nil - }, ctx.Done()) + }) if err != nil { t.Errorf("failed to wait isPoolCacheSynced to be initialized as true") } @@ -182,12 +182,12 @@ func TestInformerSyncLeaseDelete(t *testing.T) { t.Errorf("failed to delete lease, %v", err) } - err = wait.PollUntil(50*time.Millisecond, func() (done bool, err error) { + err = wait.PollUntilContextCancel(ctx, 50*time.Millisecond, true, func(ctx2 context.Context) (done bool, err error) { if isPoolCacheSynced { return false, nil } return true, nil - }, ctx.Done()) + }) if err != nil { t.Errorf("unexpect err when waitting isPoolCacheSynced to be false, %v", err) } @@ -208,7 +208,7 @@ func TestIfInformerSyncLease(t *testing.T) { Namespace: namespaceInformerLease, }, Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.StringPtr("leader-yurthub"), + HolderIdentity: pointer.String("leader-yurthub"), }, }, Expect: true, @@ -221,7 +221,7 @@ func TestIfInformerSyncLease(t *testing.T) { Namespace: "kube-system", }, Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.StringPtr("other-lease"), + HolderIdentity: pointer.String("other-lease"), }, }, Expect: false, diff --git a/pkg/yurthub/yurtcoordinator/informer_lease.go b/pkg/yurthub/yurtcoordinator/informer_lease.go index ea2de15de8a..4a3c122b2e5 100644 --- a/pkg/yurthub/yurtcoordinator/informer_lease.go +++ b/pkg/yurthub/yurtcoordinator/informer_lease.go @@ -151,8 +151,8 @@ func (nl *informerLeaseTmpl) newLease(base *coordinationv1.Lease) *coordinationv Namespace: nl.leaseNamespace, }, Spec: coordinationv1.LeaseSpec{ - HolderIdentity: pointer.StringPtr(nl.holderIdentity), - LeaseDurationSeconds: pointer.Int32Ptr(nl.leaseDurationSeconds), + HolderIdentity: pointer.String(nl.holderIdentity), + LeaseDurationSeconds: pointer.Int32(nl.leaseDurationSeconds), }, } } else { diff --git a/pkg/yurtiotdock/controllers/util/tools.go b/pkg/yurtiotdock/controllers/util/tools.go index a95dc50395a..e8abca5a15d 100644 --- a/pkg/yurtiotdock/controllers/util/tools.go +++ b/pkg/yurtiotdock/controllers/util/tools.go @@ -19,7 +19,7 @@ package util import ( "context" "fmt" - "io/ioutil" + "os" "strings" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -42,11 +42,11 @@ func GetNodePool(cfg *rest.Config) (string, error) { return nodePool, err } - bn, err := ioutil.ReadFile(PODHOSTNAME) + bn, err := os.ReadFile(PODHOSTNAME) if err != nil { return nodePool, fmt.Errorf("Read file %s fail: %v", PODHOSTNAME, err) } - bns, err := ioutil.ReadFile(PODNAMESPACE) + bns, err := os.ReadFile(PODNAMESPACE) if err != nil { return nodePool, fmt.Errorf("Read file %s fail: %v", PODNAMESPACE, err) } diff --git a/pkg/yurtmanager/controller/csrapprover/csrapprover_controller.go b/pkg/yurtmanager/controller/csrapprover/csrapprover_controller.go index 6cadc317d33..bf83a9a9f54 100644 --- a/pkg/yurtmanager/controller/csrapprover/csrapprover_controller.go +++ b/pkg/yurtmanager/controller/csrapprover/csrapprover_controller.go @@ -27,12 +27,10 @@ import ( certificatesv1beta1 "k8s.io/api/certificates/v1beta1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -54,12 +52,12 @@ var ( yurtCsr = fmt.Sprintf("%s-csr", strings.TrimRightFunc(projectinfo.GetProjectPrefix(), func(c rune) bool { return c == '-' })) yurtHubNodeCertOrgPrefix = "openyurt:tenant:" - clientRequiredUsages = sets.NewString( + clientRequiredUsages = sets.New( string(certificatesv1.UsageDigitalSignature), string(certificatesv1.UsageKeyEncipherment), string(certificatesv1.UsageClientAuth)) - serverRequiredUsages = sets.NewString( + serverRequiredUsages = sets.New( string(certificatesv1.UsageDigitalSignature), string(certificatesv1.UsageKeyEncipherment), string(certificatesv1.UsageServerAuth)) @@ -96,7 +94,11 @@ type csrRecognizer struct { // Add creates a new CsrApprover Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(_ context.Context, cfg *appconfig.CompletedConfig, mgr manager.Manager) error { - r := &ReconcileCsrApprover{} + r, err := NewReconcileCsrApprover(mgr) + if err != nil { + return err + } + // Create a new controller c, err := controller.New(names.CsrApproverController, mgr, controller.Options{ Reconciler: r, MaxConcurrentReconciles: int(cfg.Config.ComponentConfig.CsrApproverController.ConcurrentCsrApproverWorkers), @@ -107,9 +109,9 @@ func Add(_ context.Context, cfg *appconfig.CompletedConfig, mgr manager.Manager) // Watch for csr changes if r.csrV1Supported { - return c.Watch(&source.Kind{Type: &certificatesv1.CertificateSigningRequest{}}, &handler.EnqueueRequestForObject{}) + return c.Watch(source.Kind(mgr.GetCache(), &certificatesv1.CertificateSigningRequest{}), &handler.EnqueueRequestForObject{}) } else { - return c.Watch(&source.Kind{Type: &certificatesv1beta1.CertificateSigningRequest{}}, &handler.EnqueueRequestForObject{}) + return c.Watch(source.Kind(mgr.GetCache(), &certificatesv1beta1.CertificateSigningRequest{}), &handler.EnqueueRequestForObject{}) } } @@ -122,12 +124,18 @@ type ReconcileCsrApprover struct { csrApproverClient kubernetes.Interface } -func (r *ReconcileCsrApprover) InjectClient(c client.Client) error { - r.Client = c - return nil -} +func NewReconcileCsrApprover(mgr manager.Manager) (*ReconcileCsrApprover, error) { + r := &ReconcileCsrApprover{ + Client: mgr.GetClient(), + } + client, err := kubernetes.NewForConfig(mgr.GetConfig()) + if err != nil { + klog.Errorf("could not create kube client, %v", err) + return nil, err + } + r.csrApproverClient = client -func (r *ReconcileCsrApprover) InjectMapper(mapper meta.RESTMapper) error { + mapper := mgr.GetRESTMapper() if gvk, err := mapper.KindFor(csrV1Resource); err != nil { klog.Errorf("v1.CertificateSigningRequest is not supported, %v", err) r.csrV1Supported = false @@ -136,17 +144,7 @@ func (r *ReconcileCsrApprover) InjectMapper(mapper meta.RESTMapper) error { r.csrV1Supported = true } - return nil -} - -func (r *ReconcileCsrApprover) InjectConfig(cfg *rest.Config) error { - client, err := kubernetes.NewForConfig(cfg) - if err != nil { - klog.Errorf("could not create kube client, %v", err) - return err - } - r.csrApproverClient = client - return nil + return r, nil } // +kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests,verbs=get;list;watch @@ -442,8 +440,8 @@ func isYurtCoordinatorClientCert(csr *certificatesv1.CertificateSigningRequest, return true } -func usagesToSet(usages []certificatesv1.KeyUsage) sets.String { - result := sets.NewString() +func usagesToSet(usages []certificatesv1.KeyUsage) sets.Set[string] { + result := sets.New[string]() for _, usage := range usages { result.Insert(string(usage)) } diff --git a/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller.go b/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller.go index f6ab8c43d20..6e8ab580622 100644 --- a/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller.go +++ b/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller.go @@ -33,7 +33,6 @@ import ( intstrutil "k8s.io/apimachinery/pkg/util/intstr" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" @@ -96,7 +95,11 @@ func Format(format string, args ...interface{}) string { // and Start it when the Manager is Started. func Add(ctx context.Context, c *appconfig.CompletedConfig, mgr manager.Manager) error { klog.Infof("daemonupdater-controller add controller %s", controllerKind.String()) - return add(mgr, c, newReconciler(c, mgr)) + r, err := newReconciler(c, mgr) + if err != nil { + return err + } + return add(mgr, c, r) } var _ reconcile.Reconciler = &ReconcileDaemonpodupdater{} @@ -110,26 +113,25 @@ type ReconcileDaemonpodupdater struct { } // newReconciler returns a new reconcile.Reconciler -func newReconciler(_ *appconfig.CompletedConfig, mgr manager.Manager) reconcile.Reconciler { - return &ReconcileDaemonpodupdater{ +func newReconciler(_ *appconfig.CompletedConfig, mgr manager.Manager) (reconcile.Reconciler, error) { + r := &ReconcileDaemonpodupdater{ Client: mgr.GetClient(), expectations: k8sutil.NewControllerExpectations(), recorder: mgr.GetEventRecorderFor(names.DaemonPodUpdaterController), } -} -func (r *ReconcileDaemonpodupdater) InjectConfig(cfg *rest.Config) error { - c, err := kubernetes.NewForConfig(cfg) + c, err := kubernetes.NewForConfig(mgr.GetConfig()) if err != nil { klog.Errorf("could not create kube client, %v", err) - return err + return nil, err } // Use PodControlInterface to delete pods, which is convenient for testing r.podControl = k8sutil.RealPodControl{ KubeClient: c, Recorder: r.recorder, } - return nil + + return r, nil } // add adds a new Controller to mgr with r as the reconcile.Reconciler @@ -156,14 +158,14 @@ func add(mgr manager.Manager, cfg *appconfig.CompletedConfig, r reconcile.Reconc }, } - if err := c.Watch(&source.Kind{Type: &appsv1.DaemonSet{}}, &handler.EnqueueRequestForObject{}, daemonsetUpdatePredicate); err != nil { + if err := c.Watch(source.Kind(mgr.GetCache(), &appsv1.DaemonSet{}), &handler.EnqueueRequestForObject{}, daemonsetUpdatePredicate); err != nil { return err } // 2. Watch for deletion of pods. The reason we watch is that we don't want a daemon set to delete // more pods until all the effects (expectations) of a daemon set's delete have been observed. updater := r.(*ReconcileDaemonpodupdater) - if err := c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.Funcs{ + if err := c.Watch(source.Kind(mgr.GetCache(), &corev1.Pod{}), &handler.Funcs{ DeleteFunc: updater.deletePod, }); err != nil { return err @@ -254,7 +256,7 @@ func (r *ReconcileDaemonpodupdater) Reconcile(_ context.Context, request reconci return reconcile.Result{}, nil } -func (r *ReconcileDaemonpodupdater) deletePod(evt event.DeleteEvent, _ workqueue.RateLimitingInterface) { +func (r *ReconcileDaemonpodupdater) deletePod(ctx context.Context, evt event.DeleteEvent, _ workqueue.RateLimitingInterface) { pod, ok := evt.Object.(*corev1.Pod) if !ok { utilruntime.HandleError(fmt.Errorf("deletepod could not deal with object that is not a pod %#v", evt.Object)) diff --git a/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller_test.go b/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller_test.go index 744fd618a0d..e7aeef84c10 100644 --- a/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller_test.go +++ b/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller_test.go @@ -207,7 +207,7 @@ type tCase struct { func TestDaemonsetPodUpdater(t *testing.T) { tcases := []tCase{ { - name: "failed with not OnDelete strategy", + name: "not OnDelete strategy", onDelete: false, strategy: "Auto", nodeNum: 3, diff --git a/pkg/yurtmanager/controller/daemonpodupdater/kubernetes/controller_utils_test.go b/pkg/yurtmanager/controller/daemonpodupdater/kubernetes/controller_utils_test.go index 406141f8daf..7d81aef0fa4 100644 --- a/pkg/yurtmanager/controller/daemonpodupdater/kubernetes/controller_utils_test.go +++ b/pkg/yurtmanager/controller/daemonpodupdater/kubernetes/controller_utils_test.go @@ -33,7 +33,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/clock" "k8s.io/apimachinery/pkg/util/uuid" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" @@ -42,12 +41,13 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" utiltesting "k8s.io/client-go/util/testing" + testingclock "k8s.io/utils/clock/testing" ) // NewFakeControllerExpectationsLookup creates a fake store for PodExpectations. -func NewFakeControllerExpectationsLookup(ttl time.Duration) (*ControllerExpectations, *clock.FakeClock) { +func NewFakeControllerExpectationsLookup(ttl time.Duration) (*ControllerExpectations, *testingclock.FakeClock) { fakeTime := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) - fakeClock := clock.NewFakeClock(fakeTime) + fakeClock := testingclock.NewFakeClock(fakeTime) ttlPolicy := &cache.TTLPolicy{TTL: ttl, Clock: fakeClock} ttlStore := cache.NewFakeExpirationStore( ExpKeyFunc, nil, ttlPolicy, fakeClock) diff --git a/pkg/yurtmanager/controller/daemonpodupdater/util.go b/pkg/yurtmanager/controller/daemonpodupdater/util.go index 5c70c5380e5..7a4f447b79a 100644 --- a/pkg/yurtmanager/controller/daemonpodupdater/util.go +++ b/pkg/yurtmanager/controller/daemonpodupdater/util.go @@ -130,7 +130,7 @@ func SetPodUpgradeCondition(c client.Client, ds *appsv1.DaemonSet, pod *corev1.P } if change := podutil.UpdatePodCondition(&pod.Status, cond); change { - if err := c.Status().Update(context.TODO(), pod, &client.UpdateOptions{}); err != nil { + if err := c.Status().Update(context.TODO(), pod, &client.SubResourceUpdateOptions{}); err != nil { return err } klog.Infof("set pod %q condition PodNeedUpgrade to %v", pod.Name, !isUpdatable) diff --git a/pkg/yurtmanager/controller/internal/controller/controller.go b/pkg/yurtmanager/controller/internal/controller/controller.go index af6a2302cf7..3a96018efed 100644 --- a/pkg/yurtmanager/controller/internal/controller/controller.go +++ b/pkg/yurtmanager/controller/internal/controller/controller.go @@ -30,12 +30,9 @@ import ( "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/source" ) -var _ inject.Injector = &Controller{} - // Controller implements controller.Controller. type Controller struct { // Name is used to uniquely identify a Controller in tracing, logging and monitoring. Name is required. @@ -50,10 +47,6 @@ type Controller struct { // the Queue for processing Queue workqueue.RateLimitingInterface - // SetFields is used to inject dependencies into other objects such as Sources, EventHandlers and Predicates - // Deprecated: the caller should handle injected fields itself. - SetFields func(i interface{}) error - // mu is used to synchronize Controller setup mu sync.Mutex @@ -75,7 +68,10 @@ type Controller struct { startWatches []watchDescription // RecoverPanic indicates whether the panic caused by reconcile should be recovered. - RecoverPanic bool + RecoverPanic *bool + + // LeaderElected indicates whether the controller is leader elected or always running. + LeaderElected *bool } // watchDescription contains all the information necessary to start a watch. @@ -89,20 +85,6 @@ type watchDescription struct { func (c *Controller) Watch(src source.Source, evthdler handler.EventHandler, prct ...predicate.Predicate) error { c.mu.Lock() defer c.mu.Unlock() - - // Inject Cache into arguments - if err := c.SetFields(src); err != nil { - return err - } - if err := c.SetFields(evthdler); err != nil { - return err - } - for _, pr := range prct { - if err := c.SetFields(pr); err != nil { - return err - } - } - // Controller hasn't started yet, store the watches locally and return. // // These watches are going to be held on the controller struct until the manager or user calls Start(...). @@ -115,6 +97,14 @@ func (c *Controller) Watch(src source.Source, evthdler handler.EventHandler, prc return src.Start(c.ctx, evthdler, c.Queue, prct...) } +// NeedLeaderElection implements the manager.LeaderElectionRunnable interface. +func (c *Controller) NeedLeaderElection() bool { + if c.LeaderElected == nil { + return true + } + return *c.LeaderElected +} + // Start implements controller.Controller. func (c *Controller) Start(ctx context.Context) error { // use an IIFE to get proper lock handling @@ -197,14 +187,8 @@ func (c *Controller) Start(ctx context.Context) error { return nil } -// InjectFunc implement SetFields.Injector. -func (c *Controller) InjectFunc(f inject.Func) error { - c.SetFields = f - return nil -} - func (c *Controller) WaitForStarted(ctx context.Context) bool { - err := wait.PollImmediateUntil(200*time.Millisecond, func() (bool, error) { + err := wait.PollUntilContextCancel(ctx, 200*time.Millisecond, true, func(ctx context.Context) (bool, error) { c.mu.Lock() started := c.Started c.mu.Unlock() @@ -212,7 +196,7 @@ func (c *Controller) WaitForStarted(ctx context.Context) bool { return false, nil } return true, nil - }, ctx.Done()) + }) if err != nil { klog.V(2).InfoS("could not start %s controller , %v", c.Name, err) return false diff --git a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/event_handler.go b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/event_handler.go index 532128b284d..7d9d2dadb99 100644 --- a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/event_handler.go +++ b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/event_handler.go @@ -36,16 +36,16 @@ import ( func NewPoolServiceEventHandler() handler.EventHandler { return handler.Funcs{ - CreateFunc: func(event event.CreateEvent, limitingInterface workqueue.RateLimitingInterface) { + CreateFunc: func(ctx context.Context, event event.CreateEvent, limitingInterface workqueue.RateLimitingInterface) { handlePoolServiceNormal(event.Object, limitingInterface) }, - UpdateFunc: func(updateEvent event.UpdateEvent, limitingInterface workqueue.RateLimitingInterface) { + UpdateFunc: func(ctx context.Context, updateEvent event.UpdateEvent, limitingInterface workqueue.RateLimitingInterface) { handlePoolServiceUpdate(updateEvent.ObjectOld, updateEvent.ObjectNew, limitingInterface) }, - DeleteFunc: func(deleteEvent event.DeleteEvent, limitingInterface workqueue.RateLimitingInterface) { + DeleteFunc: func(ctx context.Context, deleteEvent event.DeleteEvent, limitingInterface workqueue.RateLimitingInterface) { handlePoolServiceNormal(deleteEvent.Object, limitingInterface) }, - GenericFunc: func(genericEvent event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) { + GenericFunc: func(ctx context.Context, genericEvent event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) { handlePoolServiceNormal(genericEvent.Object, limitingInterface) }, } @@ -95,16 +95,16 @@ func handlePoolServiceUpdate(oldObject, newObject client.Object, q workqueue.Rat func NewNodePoolEventHandler(c client.Client) handler.EventHandler { return handler.Funcs{ - CreateFunc: func(createEvent event.CreateEvent, limitingInterface workqueue.RateLimitingInterface) { + CreateFunc: func(ctx context.Context, createEvent event.CreateEvent, limitingInterface workqueue.RateLimitingInterface) { allLoadBalancerSetServicesEnqueue(c, limitingInterface) }, - UpdateFunc: func(updateEvent event.UpdateEvent, limitingInterface workqueue.RateLimitingInterface) { + UpdateFunc: func(ctx context.Context, updateEvent event.UpdateEvent, limitingInterface workqueue.RateLimitingInterface) { allLoadBalancerSetServicesEnqueue(c, limitingInterface) }, - DeleteFunc: func(deleteEvent event.DeleteEvent, limitingInterface workqueue.RateLimitingInterface) { + DeleteFunc: func(ctx context.Context, deleteEvent event.DeleteEvent, limitingInterface workqueue.RateLimitingInterface) { nodePoolRelatedServiceEnqueue(c, deleteEvent.Object, limitingInterface) }, - GenericFunc: func(genericEvent event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) { + GenericFunc: func(ctx context.Context, genericEvent event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) { nodePoolRelatedServiceEnqueue(c, genericEvent.Object, limitingInterface) }, } diff --git a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/event_handler_test.go b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/event_handler_test.go index c383adbc532..4c462b75a13 100644 --- a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/event_handler_test.go +++ b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/event_handler_test.go @@ -17,6 +17,7 @@ limitations under the License. package loadbalancerset import ( + "context" "testing" corev1 "k8s.io/api/core/v1" @@ -35,9 +36,9 @@ func TestPoolServiceEventHandler(t *testing.T) { f := NewPoolServiceEventHandler() t.Run("create pool service", func(t *testing.T) { ps := newPoolServiceWithServiceNameAndNodepoolName(mockServiceName, "np123") - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "pool_services") + q := workqueue.NewRateLimitingQueueWithConfig(workqueue.DefaultControllerRateLimiter(), workqueue.RateLimitingQueueConfig{Name: "pool_services"}) - f.Create(event.CreateEvent{ + f.Create(context.Background(), event.CreateEvent{ Object: ps, }, q) @@ -47,9 +48,9 @@ func TestPoolServiceEventHandler(t *testing.T) { t.Run("create pool service not service name", func(t *testing.T) { ps := newPoolServiceWithServiceNameAndNodepoolName(mockServiceName, "np123") delete(ps.Labels, network.LabelServiceName) - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "pool_services") + q := workqueue.NewRateLimitingQueueWithConfig(workqueue.DefaultControllerRateLimiter(), workqueue.RateLimitingQueueConfig{Name: "pool_services"}) - f.Create(event.CreateEvent{ + f.Create(context.Background(), event.CreateEvent{ Object: ps, }, q) assertAndDoneQueue(t, q, []string{}) @@ -59,9 +60,9 @@ func TestPoolServiceEventHandler(t *testing.T) { oldPs := newPoolServiceWithServiceNameAndNodepoolName("mock1", "np123") newPs := newPoolServiceWithServiceNameAndNodepoolName("mock2", "np123") - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "pool_services") + q := workqueue.NewRateLimitingQueueWithConfig(workqueue.DefaultControllerRateLimiter(), workqueue.RateLimitingQueueConfig{Name: "pool_services"}) - f.Update(event.UpdateEvent{ObjectOld: oldPs, ObjectNew: newPs}, q) + f.Update(context.Background(), event.UpdateEvent{ObjectOld: oldPs, ObjectNew: newPs}, q) assertAndDoneQueue(t, q, []string{ v1.NamespaceDefault + "/" + "mock1", @@ -74,9 +75,9 @@ func TestPoolServiceEventHandler(t *testing.T) { newPs := newPoolServiceWithServiceNameAndNodepoolName("mock1", "np123") delete(newPs.Labels, network.LabelServiceName) - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "pool_services") + q := workqueue.NewRateLimitingQueueWithConfig(workqueue.DefaultControllerRateLimiter(), workqueue.RateLimitingQueueConfig{Name: "pool_services"}) - f.Update(event.UpdateEvent{ObjectOld: oldPs, ObjectNew: newPs}, q) + f.Update(context.Background(), event.UpdateEvent{ObjectOld: oldPs, ObjectNew: newPs}, q) assertAndDoneQueue(t, q, []string{v1.NamespaceDefault + "/" + "mock1"}) }) @@ -86,9 +87,9 @@ func TestPoolServiceEventHandler(t *testing.T) { newPs := newPoolServiceWithServiceNameAndNodepoolName(mockServiceName, "np123") newPs.Annotations = map[string]string{"test": "app"} - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "pool_services") + q := workqueue.NewRateLimitingQueueWithConfig(workqueue.DefaultControllerRateLimiter(), workqueue.RateLimitingQueueConfig{Name: "pool_services"}) - f.Update(event.UpdateEvent{ + f.Update(context.Background(), event.UpdateEvent{ ObjectOld: oldPs, ObjectNew: newPs, }, q) @@ -150,12 +151,12 @@ func TestNodePoolEventHandler(t *testing.T) { c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(svc1).WithObjects(svc2).Build() f := NewNodePoolEventHandler(c) - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "pool_services") + q := workqueue.NewRateLimitingQueueWithConfig(workqueue.DefaultControllerRateLimiter(), workqueue.RateLimitingQueueConfig{Name: "pool_services"}) - f.Create(event.CreateEvent{Object: np}, q) + f.Create(context.Background(), event.CreateEvent{Object: np}, q) assertAndDoneQueue(t, q, []string{v1.NamespaceDefault + "/" + mockServiceName}) - f.Update(event.UpdateEvent{ObjectOld: np, ObjectNew: np}, q) + f.Update(context.Background(), event.UpdateEvent{ObjectOld: np, ObjectNew: np}, q) assertAndDoneQueue(t, q, []string{v1.NamespaceDefault + "/" + mockServiceName}) }) @@ -169,16 +170,16 @@ func TestNodePoolEventHandler(t *testing.T) { f := NewNodePoolEventHandler(c) np := newNodepool("np123", "name=np123,app=deploy") - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "pool_services") + q := workqueue.NewRateLimitingQueueWithConfig(workqueue.DefaultControllerRateLimiter(), workqueue.RateLimitingQueueConfig{Name: "pool_services"}) - f.Delete(event.DeleteEvent{Object: np}, q) + f.Delete(context.Background(), event.DeleteEvent{Object: np}, q) assertAndDoneQueue(t, q, []string{ v1.NamespaceDefault + "/" + "mock1", v1.NamespaceDefault + "/" + "mock2", v1.NamespaceDefault + "/" + "mock3", }) - f.Generic(event.GenericEvent{Object: np}, q) + f.Generic(context.Background(), event.GenericEvent{Object: np}, q) assertAndDoneQueue(t, q, []string{ v1.NamespaceDefault + "/" + "mock1", v1.NamespaceDefault + "/" + "mock2", @@ -193,18 +194,18 @@ func TestNodePoolEventHandler(t *testing.T) { f := NewNodePoolEventHandler(c) np := newNodepool("np234", "name=np234,app=deploy") - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "pool_services") + q := workqueue.NewRateLimitingQueueWithConfig(workqueue.DefaultControllerRateLimiter(), workqueue.RateLimitingQueueConfig{Name: "pool_services"}) - f.Create(event.CreateEvent{Object: np}, q) + f.Create(context.Background(), event.CreateEvent{Object: np}, q) assertAndDoneQueue(t, q, []string{}) - f.Update(event.UpdateEvent{ObjectOld: np, ObjectNew: np}, q) + f.Update(context.Background(), event.UpdateEvent{ObjectOld: np, ObjectNew: np}, q) assertAndDoneQueue(t, q, []string{}) - f.Delete(event.DeleteEvent{Object: np}, q) + f.Delete(context.Background(), event.DeleteEvent{Object: np}, q) assertAndDoneQueue(t, q, []string{}) - f.Generic(event.GenericEvent{Object: np}, q) + f.Generic(context.Background(), event.GenericEvent{Object: np}, q) assertAndDoneQueue(t, q, []string{}) }) diff --git a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/loadbalancerset_controller.go b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/loadbalancerset_controller.go index c5bd0d5444a..b92dd9fca23 100644 --- a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/loadbalancerset_controller.go +++ b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/loadbalancerset_controller.go @@ -113,17 +113,17 @@ func add(mgr manager.Manager, cfg *appconfig.CompletedConfig, r reconcile.Reconc } // Watch for changes to PoolService - err = c.Watch(&source.Kind{Type: &corev1.Service{}}, &handler.EnqueueRequestForObject{}, NewServicePredicated()) + err = c.Watch(source.Kind(mgr.GetCache(), &corev1.Service{}), &handler.EnqueueRequestForObject{}, NewServicePredicated()) if err != nil { return err } - err = c.Watch(&source.Kind{Type: &netv1alpha1.PoolService{}}, NewPoolServiceEventHandler(), NewPoolServicePredicated()) + err = c.Watch(source.Kind(mgr.GetCache(), &netv1alpha1.PoolService{}), NewPoolServiceEventHandler(), NewPoolServicePredicated()) if err != nil { return err } - err = c.Watch(&source.Kind{Type: &v1beta1.NodePool{}}, NewNodePoolEventHandler(mgr.GetClient()), NewNodePoolPredicated()) + err = c.Watch(source.Kind(mgr.GetCache(), &v1beta1.NodePool{}), NewNodePoolEventHandler(mgr.GetClient()), NewNodePoolPredicated()) if err != nil { return err } @@ -497,13 +497,16 @@ func (r *ReconcileLoadBalancerSet) syncService(svc *corev1.Service) error { } func (r *ReconcileLoadBalancerSet) compareAndUpdateService(svc *corev1.Service, labels, annotations map[string]string, lbStatus corev1.LoadBalancerStatus) error { - isUpdatedAnnotations := compareAndUpdateServiceAnnotations(svc, annotations) isUpdatedLbStatus := compareAndUpdateServiceLbStatus(svc, lbStatus) - isUpdatedLabels := compareAndUpdateServiceLabels(svc, labels) + if isUpdatedLbStatus { + return r.Status().Update(context.Background(), svc) + } - if !isUpdatedLbStatus && !isUpdatedAnnotations && !isUpdatedLabels { - return nil + isUpdatedAnnotations := compareAndUpdateServiceAnnotations(svc, annotations) + isUpdatedLabels := compareAndUpdateServiceLabels(svc, labels) + if isUpdatedAnnotations || isUpdatedLabels { + return r.Update(context.Background(), svc, &client.UpdateOptions{}) } - return r.Status().Update(context.Background(), svc) + return nil } diff --git a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/loadbalancerset_controller_test.go b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/loadbalancerset_controller_test.go index b2e984dad47..963d7853ec0 100644 --- a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/loadbalancerset_controller_test.go +++ b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/loadbalancerset_controller_test.go @@ -194,8 +194,11 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { Client: c, } + // call Reconcile twice _, err := rc.Reconcile(context.Background(), newReconcileRequest(v1.NamespaceDefault, mockServiceName)) assertErrNil(t, err) + _, err = rc.Reconcile(context.Background(), newReconcileRequest(v1.NamespaceDefault, mockServiceName)) + assertErrNil(t, err) newSvc := &corev1.Service{} c.Get(context.Background(), types.NamespacedName{ @@ -250,7 +253,7 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { np2 := newNodepool("np234", "name=np234,app=deploy") ps1 := newPoolService(v1.NamespaceDefault, "np123", nil, map[string]string{"lb-id": "lb34567"}, nil) - svc.Annotations["lb-id"] = "lb23456,lb34567" + svc.Annotations[aggregateKeyPrefix+"lb-id"] = "lb23456,lb34567" c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(svc).WithObjects(np1).WithObjects(np2). WithObjects(ps1).Build() @@ -259,8 +262,11 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { Client: c, } + // call Reconcile twice _, err := rc.Reconcile(context.Background(), newReconcileRequest(v1.NamespaceDefault, mockServiceName)) assertErrNil(t, err) + _, err = rc.Reconcile(context.Background(), newReconcileRequest(v1.NamespaceDefault, mockServiceName)) + assertErrNil(t, err) newSvc := &corev1.Service{} c.Get(context.Background(), types.NamespacedName{ @@ -288,8 +294,11 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { Client: c, } + // call Reconcile twice _, err := rc.Reconcile(context.Background(), newReconcileRequest(v1.NamespaceDefault, mockServiceName)) assertErrNil(t, err) + _, err = rc.Reconcile(context.Background(), newReconcileRequest(v1.NamespaceDefault, mockServiceName)) + assertErrNil(t, err) newSvc := &corev1.Service{} c.Get(context.Background(), types.NamespacedName{ @@ -775,9 +784,10 @@ func newNodepool(name string, labelStr string) *v1beta1.NodePool { APIVersion: "apps.openyurt.io/v1beta1", }, ObjectMeta: v1.ObjectMeta{ - Labels: labels, - Name: name, - UID: mockNodePoolUid, + Labels: labels, + Name: name, + UID: mockNodePoolUid, + Finalizers: []string{"openyurt.io/nodepool"}, }, } } diff --git a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/map_utils.go b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/map_utils.go index ea9a4b63597..0719e6f52da 100644 --- a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/map_utils.go +++ b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/map_utils.go @@ -32,14 +32,14 @@ func aggregatedMaps(elems ...map[string]string) map[string]string { return mapSetToMapString(aggregatedSet) } -func mergeMapListToMapSet(elems ...map[string]string) map[string]sets.String { - aggregatedSet := make(map[string]sets.String) +func mergeMapListToMapSet(elems ...map[string]string) map[string]sets.Set[string] { + aggregatedSet := make(map[string]sets.Set[string]) for _, elem := range elems { for key, value := range elem { aggregatedKey := aggregateKeyPrefix + key if _, exists := aggregatedSet[aggregatedKey]; !exists { - aggregatedSet[aggregatedKey] = sets.NewString() + aggregatedSet[aggregatedKey] = sets.New[string]() } aggregatedSet[aggregatedKey].Insert(value) } @@ -47,10 +47,10 @@ func mergeMapListToMapSet(elems ...map[string]string) map[string]sets.String { return aggregatedSet } -func mapSetToMapString(aggregatedSet map[string]sets.String) map[string]string { +func mapSetToMapString(aggregatedSet map[string]sets.Set[string]) map[string]string { result := make(map[string]string) for key, value := range aggregatedSet { - result[key] = strings.Join(value.List(), ",") + result[key] = strings.Join(sets.List(value), ",") } return result } diff --git a/pkg/yurtmanager/controller/nodebucket/nodebucket_controller.go b/pkg/yurtmanager/controller/nodebucket/nodebucket_controller.go index 33c1d1838b8..9ba36b0bce5 100644 --- a/pkg/yurtmanager/controller/nodebucket/nodebucket_controller.go +++ b/pkg/yurtmanager/controller/nodebucket/nodebucket_controller.go @@ -24,7 +24,6 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/rand" @@ -69,6 +68,7 @@ func Format(format string, args ...interface{}) string { func Add(_ context.Context, cfg *appconfig.CompletedConfig, mgr manager.Manager) error { klog.Infof(Format("nodebucket-controller add controller %s", controllerResource.String())) r := &ReconcileNodeBucket{ + Client: mgr.GetClient(), maxNodesPerBucket: int(cfg.ComponentConfig.NodeBucketController.MaxNodesPerBucket), } @@ -80,21 +80,19 @@ func Add(_ context.Context, cfg *appconfig.CompletedConfig, mgr manager.Manager) return err } - if _, err := r.mapper.KindFor(controllerResource); err != nil { + if _, err := mgr.GetRESTMapper().KindFor(controllerResource); err != nil { klog.Infof("resource %s doesn't exist", controllerResource.String()) return err } // Watch for changes to NodeBucket - if err = c.Watch(&source.Kind{Type: &appsv1alpha1.NodeBucket{}}, &handler.EnqueueRequestForOwner{ - OwnerType: &appsv1beta1.NodePool{}, - IsController: true, - }); err != nil { + if err = c.Watch(source.Kind(mgr.GetCache(), &appsv1alpha1.NodeBucket{}), + handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &appsv1beta1.NodePool{}, handler.OnlyControllerOwner())); err != nil { return err } // Watch nodepool create for nodebucket - if err = c.Watch(&source.Kind{Type: &appsv1beta1.NodePool{}}, &handler.EnqueueRequestForObject{}, predicate.Funcs{ + if err = c.Watch(source.Kind(mgr.GetCache(), &appsv1beta1.NodePool{}), &handler.EnqueueRequestForObject{}, predicate.Funcs{ CreateFunc: func(createEvent event.CreateEvent) bool { return true }, @@ -138,7 +136,7 @@ func Add(_ context.Context, cfg *appconfig.CompletedConfig, mgr manager.Manager) }, } - reconcilePool := handler.EnqueueRequestsFromMapFunc(func(obj client.Object) []reconcile.Request { + reconcilePool := handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request { node, ok := obj.(*v1.Node) if !ok { return []reconcile.Request{} @@ -154,7 +152,7 @@ func Add(_ context.Context, cfg *appconfig.CompletedConfig, mgr manager.Manager) }) // Watch for changes to Node - if err = c.Watch(&source.Kind{Type: &v1.Node{}}, reconcilePool, nodePredicate); err != nil { + if err = c.Watch(source.Kind(mgr.GetCache(), &v1.Node{}), reconcilePool, nodePredicate); err != nil { return err } return nil @@ -162,20 +160,9 @@ func Add(_ context.Context, cfg *appconfig.CompletedConfig, mgr manager.Manager) var _ reconcile.Reconciler = &ReconcileNodeBucket{} -func (r *ReconcileNodeBucket) InjectClient(c client.Client) error { - r.Client = c - return nil -} - -func (r *ReconcileNodeBucket) InjectMapper(mapper meta.RESTMapper) error { - r.mapper = mapper - return nil -} - // ReconcileNodeBucket reconciles a NodeBucket object type ReconcileNodeBucket struct { client.Client - mapper meta.RESTMapper maxNodesPerBucket int } @@ -206,7 +193,7 @@ func (r *ReconcileNodeBucket) Reconcile(ctx context.Context, request reconcile.R })); err != nil { return reconcile.Result{}, client.IgnoreNotFound(err) } - desiredNodeSet := sets.String{} + desiredNodeSet := sets.Set[string]{} for i := range currentNodeList.Items { desiredNodeSet.Insert(currentNodeList.Items[i].Name) } @@ -234,12 +221,12 @@ func (r *ReconcileNodeBucket) Reconcile(ctx context.Context, request reconcile.R func (r *ReconcileNodeBucket) reconcileNodeBuckets( pool *appsv1beta1.NodePool, - desiredNodeSet sets.String, + desiredNodeSet sets.Set[string], buckets *appsv1alpha1.NodeBucketList, ) ([]*appsv1alpha1.NodeBucket, []*appsv1alpha1.NodeBucket, []*appsv1alpha1.NodeBucket, []*appsv1alpha1.NodeBucket) { bucketsUnchanged, bucketsToUpdate, bucketsToDelete, unFilledNodeSet := resolveExistingBuckets(buckets, desiredNodeSet) klog.V(4).Infof("reconcileNodeBuckets for pool(%s), len(bucketsUnchanged)=%d, len(bucketsToUpdate)=%d, len(bucketsToDelete)=%d, unFilledNodeSet=%v", - pool.Name, len(bucketsUnchanged), len(bucketsToUpdate), len(bucketsToDelete), unFilledNodeSet.List()) + pool.Name, len(bucketsUnchanged), len(bucketsToUpdate), len(bucketsToDelete), unFilledNodeSet.UnsortedList()) // If we still have unfilled nodes to add and buckets marked for update, // iterate through the buckets and fill them up with the unfilled nodes. @@ -253,7 +240,7 @@ func (r *ReconcileNodeBucket) reconcileNodeBuckets( } } klog.V(4).Infof("reconcileNodeBuckets for pool(%s) after filling bucketsToUpdate, len(bucketsUnchanged)=%d, len(bucketsToUpdate)=%d, len(bucketsToDelete)=%d, unFilledNodeSet=%v", - pool.Name, len(bucketsUnchanged), len(bucketsToUpdate), len(bucketsToDelete), unFilledNodeSet.List()) + pool.Name, len(bucketsUnchanged), len(bucketsToUpdate), len(bucketsToDelete), unFilledNodeSet.UnsortedList()) // If there are still unfilled nodes left at this point, we try to fit the nodes in a single existing buckets. // If there are no buckets with that capacity, we create new buckets for the nodes. @@ -282,13 +269,13 @@ func (r *ReconcileNodeBucket) reconcileNodeBuckets( } } klog.V(4).Infof("reconcileNodeBuckets for pool(%s) after filling bucketsUnchanged, len(bucketsUnchanged)=%d, len(bucketsToCreate)=%d len(bucketsToUpdate)=%v, len(bucketsToDelete)=%d, unFilledNodeSet=%v", - pool.Name, len(bucketsUnchanged), len(bucketsToCreate), len(bucketsToUpdate), len(bucketsToDelete), unFilledNodeSet.List()) + pool.Name, len(bucketsUnchanged), len(bucketsToCreate), len(bucketsToUpdate), len(bucketsToDelete), unFilledNodeSet.UnsortedList()) return bucketsToCreate, bucketsToUpdate, bucketsToDelete, bucketsUnchanged } // resolveExistingBuckets iterates through existing node buckets to delete nodes no longer desired and update node buckets that have changed -func resolveExistingBuckets(buckets *appsv1alpha1.NodeBucketList, desiredNodeSet sets.String) ([]*appsv1alpha1.NodeBucket, []*appsv1alpha1.NodeBucket, []*appsv1alpha1.NodeBucket, sets.String) { +func resolveExistingBuckets(buckets *appsv1alpha1.NodeBucketList, desiredNodeSet sets.Set[string]) ([]*appsv1alpha1.NodeBucket, []*appsv1alpha1.NodeBucket, []*appsv1alpha1.NodeBucket, sets.Set[string]) { bucketsUnchanged := []*appsv1alpha1.NodeBucket{} bucketsToUpdate := []*appsv1alpha1.NodeBucket{} bucketsToDelete := []*appsv1alpha1.NodeBucket{} diff --git a/pkg/yurtmanager/controller/nodebucket/nodebucket_controller_test.go b/pkg/yurtmanager/controller/nodebucket/nodebucket_controller_test.go index 72ba8d1da4a..95477f954d5 100644 --- a/pkg/yurtmanager/controller/nodebucket/nodebucket_controller_test.go +++ b/pkg/yurtmanager/controller/nodebucket/nodebucket_controller_test.go @@ -49,7 +49,7 @@ func TestReconcile(t *testing.T) { nodeBuckets []client.Object pool client.Object wantedNumberOfBuckets int - wantedNodeNames sets.String + wantedNodeNames sets.Set[string] }{ "generate one bucket": { maxNodesPerBucket: 10, @@ -77,7 +77,7 @@ func TestReconcile(t *testing.T) { }, }, wantedNumberOfBuckets: 1, - wantedNodeNames: sets.NewString("node1", "node2"), + wantedNodeNames: sets.New("node1", "node2"), }, "generate two buckets": { maxNodesPerBucket: 2, @@ -129,7 +129,7 @@ func TestReconcile(t *testing.T) { }, }, wantedNumberOfBuckets: 2, - wantedNodeNames: sets.NewString("node1", "node2", "node3", "node5"), + wantedNodeNames: sets.New("node1", "node2", "node3", "node5"), }, "update one bucket": { maxNodesPerBucket: 10, @@ -199,7 +199,7 @@ func TestReconcile(t *testing.T) { }, }, wantedNumberOfBuckets: 1, - wantedNodeNames: sets.NewString("node1", "node2", "node3", "node5"), + wantedNodeNames: sets.New("node1", "node2", "node3", "node5"), }, "update two buckets": { maxNodesPerBucket: 2, @@ -285,7 +285,7 @@ func TestReconcile(t *testing.T) { }, }, wantedNumberOfBuckets: 2, - wantedNodeNames: sets.NewString("node1", "node2", "node3", "node5"), + wantedNodeNames: sets.New("node1", "node2", "node3", "node5"), }, "create and update one bucket": { maxNodesPerBucket: 2, @@ -355,7 +355,7 @@ func TestReconcile(t *testing.T) { }, }, wantedNumberOfBuckets: 2, - wantedNodeNames: sets.NewString("node1", "node2", "node3", "node5"), + wantedNodeNames: sets.New("node1", "node2", "node3", "node5"), }, "delete and update one bucket": { maxNodesPerBucket: 2, @@ -414,7 +414,7 @@ func TestReconcile(t *testing.T) { }, }, wantedNumberOfBuckets: 1, - wantedNodeNames: sets.NewString("node1", "node2"), + wantedNodeNames: sets.New("node1", "node2"), }, "update three buckets": { maxNodesPerBucket: 2, @@ -497,7 +497,7 @@ func TestReconcile(t *testing.T) { }, }, wantedNumberOfBuckets: 3, - wantedNodeNames: sets.NewString("node1", "node2", "node3", "node4", "node5"), + wantedNodeNames: sets.New("node1", "node2", "node3", "node4", "node5"), }, "two buckets are updated": { maxNodesPerBucket: 2, @@ -583,7 +583,7 @@ func TestReconcile(t *testing.T) { }, }, wantedNumberOfBuckets: 3, - wantedNodeNames: sets.NewString("node1", "node2", "node3", "node4", "node5"), + wantedNodeNames: sets.New("node1", "node2", "node3", "node4", "node5"), }, } @@ -620,7 +620,7 @@ func TestReconcile(t *testing.T) { t.Errorf("expect %d buckets, but got %d", tc.wantedNumberOfBuckets, len(buckets.Items)) } - gotBucketNodes := sets.String{} + gotBucketNodes := sets.Set[string]{} for i := range buckets.Items { for _, node := range buckets.Items[i].Nodes { gotBucketNodes.Insert(node.Name) diff --git a/pkg/yurtmanager/controller/nodelifecycle/node_lifecycle_controller.go b/pkg/yurtmanager/controller/nodelifecycle/node_lifecycle_controller.go index 038fb98f81b..0eac1685819 100644 --- a/pkg/yurtmanager/controller/nodelifecycle/node_lifecycle_controller.go +++ b/pkg/yurtmanager/controller/nodelifecycle/node_lifecycle_controller.go @@ -330,7 +330,7 @@ func Add(ctx context.Context, cfg *appconfig.CompletedConfig, mgr manager.Manage return false }, } - c.Watch(&source.Kind{Type: &v1.Pod{}}, &handler.Funcs{}, podsPredicate) + c.Watch(source.Kind(mgr.GetCache(), &v1.Pod{}), &handler.Funcs{}, podsPredicate) nc.taintManager = scheduler.NewNoExecuteTaintManager(nc.recorder, nc.controllerRuntimeClient, nc.getPodsAssignedToNode) nodesTaintManagerPredicate := predicate.Funcs{ @@ -354,7 +354,7 @@ func Add(ctx context.Context, cfg *appconfig.CompletedConfig, mgr manager.Manage return false }, } - c.Watch(&source.Kind{Type: &v1.Node{}}, &handler.Funcs{}, nodesTaintManagerPredicate) + c.Watch(source.Kind(mgr.GetCache(), &v1.Node{}), &handler.Funcs{}, nodesTaintManagerPredicate) nodesUpdateQueuePredicate := predicate.Funcs{ CreateFunc: func(evt event.CreateEvent) bool { @@ -376,9 +376,9 @@ func Add(ctx context.Context, cfg *appconfig.CompletedConfig, mgr manager.Manage return false }, } - c.Watch(&source.Kind{Type: &v1.Node{}}, &handler.Funcs{}, nodesUpdateQueuePredicate) - c.Watch(&source.Kind{Type: &apps.DaemonSet{}}, &handler.Funcs{}) - c.Watch(&source.Kind{Type: &coordinationv1.Lease{}}, &handler.Funcs{}) + c.Watch(source.Kind(mgr.GetCache(), &v1.Node{}), &handler.Funcs{}, nodesUpdateQueuePredicate) + c.Watch(source.Kind(mgr.GetCache(), &apps.DaemonSet{}), &handler.Funcs{}) + c.Watch(source.Kind(mgr.GetCache(), &coordinationv1.Lease{}), &handler.Funcs{}) go nc.Run(ctx, c.WaitForStarted) return nil @@ -687,7 +687,7 @@ func (nc *ReconcileNodeLifeCycle) monitorNodeHealth(ctx context.Context) error { var currentReadyCondition *v1.NodeCondition node := nodes[piece].DeepCopy() - if err := wait.PollImmediate(retrySleepTime, retrySleepTime*scheduler.NodeHealthUpdateRetry, func() (bool, error) { + if err := wait.PollUntilContextTimeout(ctx, retrySleepTime, retrySleepTime*scheduler.NodeHealthUpdateRetry, true, func(ctx context.Context) (bool, error) { var err error _, observedReadyCondition, currentReadyCondition, err = nc.tryUpdateNodeHealth(ctx, node) if err == nil { @@ -960,7 +960,7 @@ func (nc *ReconcileNodeLifeCycle) tryUpdateNodeHealth(ctx context.Context, node if !apiequality.Semantic.DeepEqual(currentReadyCondition, &observedReadyCondition) { //if _, err := nc.kubeClient.CoreV1().Nodes().UpdateStatus(ctx, node, metav1.UpdateOptions{}); err != nil { - if err := nc.controllerRuntimeClient.Status().Update(ctx, node, &client.UpdateOptions{}); err != nil { + if err := nc.controllerRuntimeClient.Status().Update(ctx, node, &client.SubResourceUpdateOptions{}); err != nil { klog.ErrorS(err, "Error updating node", "node", klog.KObj(node)) return gracePeriod, observedReadyCondition, currentReadyCondition, err } diff --git a/pkg/yurtmanager/controller/nodelifecycle/node_lifecycle_controller_test.go b/pkg/yurtmanager/controller/nodelifecycle/node_lifecycle_controller_test.go index 9e16bccf07a..c9b78d09dfb 100644 --- a/pkg/yurtmanager/controller/nodelifecycle/node_lifecycle_controller_test.go +++ b/pkg/yurtmanager/controller/nodelifecycle/node_lifecycle_controller_test.go @@ -2981,16 +2981,22 @@ func TestTaintsNodeByCondition(t *testing.T) { } for _, test := range tests { - fakeNodeHandler.DelegateNodeHandler.Update(ctx, test.Node, metav1.UpdateOptions{}) + _, err := fakeNodeHandler.DelegateNodeHandler.UpdateStatus(ctx, test.Node, metav1.UpdateOptions{}) + if err != nil { + t.Errorf("unxpected error %v", err) + } //if err := nodeController.syncNodeStore(fakeNodeHandler); err != nil { // t.Errorf("unexpected error: %v", err) //} - nodeController.doNoScheduleTaintingPass(ctx, test.Node.Name) + err = nodeController.doNoScheduleTaintingPass(ctx, test.Node.Name) + if err != nil { + t.Errorf("Can't do noschedule taint, %v", err) + } //if err := nodeController.syncNodeStore(fakeNodeHandler); err != nil { // t.Errorf("unexpected error: %v", err) //} node0 := new(v1.Node) - err := nodeController.controllerRuntimeClient.Get(ctx, client.ObjectKey{Name: "node0"}, node0) + err = nodeController.controllerRuntimeClient.Get(ctx, client.ObjectKey{Name: "node0"}, node0) if err != nil { t.Errorf("Can't get current node0...") return diff --git a/pkg/yurtmanager/controller/nodelifecycle/scheduler/rate_limited_queue.go b/pkg/yurtmanager/controller/nodelifecycle/scheduler/rate_limited_queue.go index 868890dfa4f..782f91f6e30 100644 --- a/pkg/yurtmanager/controller/nodelifecycle/scheduler/rate_limited_queue.go +++ b/pkg/yurtmanager/controller/nodelifecycle/scheduler/rate_limited_queue.go @@ -81,7 +81,7 @@ func (h *TimedQueue) Pop() interface{} { type UniqueQueue struct { lock sync.Mutex queue TimedQueue - set sets.String + set sets.Set[string] } // Add a new value to the queue if it wasn't added before, or was @@ -189,7 +189,7 @@ func (q *UniqueQueue) Clear() { q.queue = make(TimedQueue, 0) } if len(q.set) > 0 { - q.set = sets.NewString() + q.set = sets.New[string]() } } @@ -207,7 +207,7 @@ func NewRateLimitedTimedQueue(limiter flowcontrol.RateLimiter) *RateLimitedTimed return &RateLimitedTimedQueue{ queue: UniqueQueue{ queue: TimedQueue{}, - set: sets.NewString(), + set: sets.New[string](), }, limiter: limiter, } diff --git a/pkg/yurtmanager/controller/nodelifecycle/scheduler/rate_limited_queue_test.go b/pkg/yurtmanager/controller/nodelifecycle/scheduler/rate_limited_queue_test.go index cd0a14b45dc..c90757afc9d 100644 --- a/pkg/yurtmanager/controller/nodelifecycle/scheduler/rate_limited_queue_test.go +++ b/pkg/yurtmanager/controller/nodelifecycle/scheduler/rate_limited_queue_test.go @@ -34,7 +34,7 @@ func CheckQueueEq(lhs []string, rhs TimedQueue) bool { return true } -func CheckSetEq(lhs, rhs sets.String) bool { +func CheckSetEq(lhs, rhs sets.Set[string]) bool { return lhs.IsSuperset(rhs) && rhs.IsSuperset(lhs) } @@ -52,7 +52,7 @@ func TestAddNode(t *testing.T) { t.Errorf("Invalid queue. Got %v, expected %v", evictor.queue.queue, queuePattern) } - setPattern := sets.NewString("first", "second", "third") + setPattern := sets.New("first", "second", "third") if len(evictor.queue.set) != len(setPattern) { t.Fatalf("Map %v should have length %d", evictor.queue.set, len(setPattern)) } @@ -83,7 +83,7 @@ func TestDelNode(t *testing.T) { t.Errorf("Invalid queue. Got %v, expected %v", evictor.queue.queue, queuePattern) } - setPattern := sets.NewString("second", "third") + setPattern := sets.New("second", "third") if len(evictor.queue.set) != len(setPattern) { t.Fatalf("Map %v should have length %d", evictor.queue.set, len(setPattern)) } @@ -105,7 +105,7 @@ func TestDelNode(t *testing.T) { t.Errorf("Invalid queue. Got %v, expected %v", evictor.queue.queue, queuePattern) } - setPattern = sets.NewString("first", "third") + setPattern = sets.New("first", "third") if len(evictor.queue.set) != len(setPattern) { t.Fatalf("Map %v should have length %d", evictor.queue.set, len(setPattern)) } @@ -127,7 +127,7 @@ func TestDelNode(t *testing.T) { t.Errorf("Invalid queue. Got %v, expected %v", evictor.queue.queue, queuePattern) } - setPattern = sets.NewString("first", "second") + setPattern = sets.New("first", "second") if len(evictor.queue.set) != len(setPattern) { t.Fatalf("Map %v should have length %d", evictor.queue.set, len(setPattern)) } @@ -143,13 +143,13 @@ func TestTry(t *testing.T) { evictor.Add("third", "33333") evictor.Remove("second") - deletedMap := sets.NewString() + deletedMap := sets.New[string]() evictor.Try(func(value TimedValue) (bool, time.Duration) { deletedMap.Insert(value.Value) return true, 0 }) - setPattern := sets.NewString("first", "third") + setPattern := sets.New("first", "third") if len(deletedMap) != len(setPattern) { t.Fatalf("Map %v should have length %d", evictor.queue.set, len(setPattern)) } @@ -311,13 +311,13 @@ func TestAddAfterTry(t *testing.T) { evictor.Add("third", "33333") evictor.Remove("second") - deletedMap := sets.NewString() + deletedMap := sets.New[string]() evictor.Try(func(value TimedValue) (bool, time.Duration) { deletedMap.Insert(value.Value) return true, 0 }) - setPattern := sets.NewString("first", "third") + setPattern := sets.New("first", "third") if len(deletedMap) != len(setPattern) { t.Fatalf("Map %v should have length %d", evictor.queue.set, len(setPattern)) } diff --git a/pkg/yurtmanager/controller/nodelifecycle/scheduler/taint_manager_test.go b/pkg/yurtmanager/controller/nodelifecycle/scheduler/taint_manager_test.go index 12224cbd392..ff8b5cfb3a9 100644 --- a/pkg/yurtmanager/controller/nodelifecycle/scheduler/taint_manager_test.go +++ b/pkg/yurtmanager/controller/nodelifecycle/scheduler/taint_manager_test.go @@ -320,7 +320,7 @@ func TestUpdatePod(t *testing.T) { if item.awaitForScheduledEviction { nsName := types.NamespacedName{Namespace: item.prevPod.Namespace, Name: item.prevPod.Name} - err := wait.PollImmediate(time.Millisecond*10, time.Second, func() (bool, error) { + err := wait.PollUntilContextTimeout(ctx, time.Millisecond*10, time.Second, true, func(ctx context.Context) (bool, error) { scheduledEviction := controller.taintEvictionQueue.GetWorkerUnsafe(nsName.String()) return scheduledEviction != nil, nil }) @@ -413,7 +413,7 @@ func TestDeleteNode(t *testing.T) { controller.NodeUpdated(NewNode("node1"), nil) // await until controller.taintedNodes is empty - err := wait.PollImmediate(10*time.Millisecond, time.Second, func() (bool, error) { + err := wait.PollUntilContextTimeout(ctx, 10*time.Millisecond, time.Second, true, func(ctx context.Context) (bool, error) { controller.taintedNodesLock.Lock() defer controller.taintedNodesLock.Unlock() _, ok := controller.taintedNodes["node1"] @@ -915,7 +915,7 @@ func verifyPodActions(t *testing.T, description string, fakeClientset *testutil. podDeleted := false // use Poll instead of PollImmediate to give some processing time to the controller that the expected // actions are likely to be already sent - err := wait.Poll(10*time.Millisecond, 5*time.Second, func() (bool, error) { + err := wait.PollUntilContextTimeout(context.Background(), 10*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { for _, action := range fakeClientset.Actions() { if action.GetVerb() == "patch" && action.GetResource().Resource == "pods" { podPatched = true diff --git a/pkg/yurtmanager/controller/nodepool/nodepool_controller.go b/pkg/yurtmanager/controller/nodepool/nodepool_controller.go index 9e8a4436370..7f909ec33d3 100644 --- a/pkg/yurtmanager/controller/nodepool/nodepool_controller.go +++ b/pkg/yurtmanager/controller/nodepool/nodepool_controller.go @@ -21,7 +21,6 @@ import ( "fmt" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" "k8s.io/client-go/tools/record" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" @@ -51,21 +50,10 @@ func Format(format string, args ...interface{}) string { // ReconcileNodePool reconciles a NodePool object type ReconcileNodePool struct { client.Client - mapper meta.RESTMapper recorder record.EventRecorder cfg poolconfig.NodePoolControllerConfiguration } -func (r *ReconcileNodePool) InjectClient(c client.Client) error { - r.Client = c - return nil -} - -func (r *ReconcileNodePool) InjectMapper(mapper meta.RESTMapper) error { - r.mapper = mapper - return nil -} - var _ reconcile.Reconciler = &ReconcileNodePool{} // Add creates a new NodePool Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller @@ -75,6 +63,7 @@ func Add(ctx context.Context, c *config.CompletedConfig, mgr manager.Manager) er r := &ReconcileNodePool{ cfg: c.ComponentConfig.NodePoolController, recorder: mgr.GetEventRecorderFor(names.NodePoolController), + Client: mgr.GetClient(), } // Create a new controller @@ -85,19 +74,19 @@ func Add(ctx context.Context, c *config.CompletedConfig, mgr manager.Manager) er return err } - if _, err := r.mapper.KindFor(controllerResource); err != nil { + if _, err := mgr.GetRESTMapper().KindFor(controllerResource); err != nil { klog.Infof("resource %s doesn't exist", controllerResource.String()) return err } // Watch for changes to NodePool - err = ctrl.Watch(&source.Kind{Type: &appsv1beta1.NodePool{}}, &handler.EnqueueRequestForObject{}) + err = ctrl.Watch(source.Kind(mgr.GetCache(), &appsv1beta1.NodePool{}), &handler.EnqueueRequestForObject{}) if err != nil { return err } // Watch for changes to Node - err = ctrl.Watch(&source.Kind{Type: &corev1.Node{}}, &EnqueueNodePoolForNode{ + err = ctrl.Watch(source.Kind(mgr.GetCache(), &corev1.Node{}), &EnqueueNodePoolForNode{ EnableSyncNodePoolConfigurations: r.cfg.EnableSyncNodePoolConfigurations, Recorder: r.recorder, }) diff --git a/pkg/yurtmanager/controller/nodepool/nodepool_controller_test.go b/pkg/yurtmanager/controller/nodepool/nodepool_controller_test.go index 41b0536708c..a489e39fd71 100644 --- a/pkg/yurtmanager/controller/nodepool/nodepool_controller_test.go +++ b/pkg/yurtmanager/controller/nodepool/nodepool_controller_test.go @@ -146,7 +146,7 @@ func TestReconcile(t *testing.T) { } apis.AddToScheme(scheme) - c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(pools...).WithObjects(nodes...).Build() + c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(pools...).WithStatusSubresource(pools...).WithObjects(nodes...).Build() testcases := map[string]struct { EnableSyncNodePoolConfigurations bool pool string diff --git a/pkg/yurtmanager/controller/nodepool/nodepool_enqueue_handlers.go b/pkg/yurtmanager/controller/nodepool/nodepool_enqueue_handlers.go index f3e533d5761..5b218f1fd5e 100644 --- a/pkg/yurtmanager/controller/nodepool/nodepool_enqueue_handlers.go +++ b/pkg/yurtmanager/controller/nodepool/nodepool_enqueue_handlers.go @@ -17,6 +17,7 @@ limitations under the License. package nodepool import ( + "context" "fmt" "reflect" @@ -38,7 +39,7 @@ type EnqueueNodePoolForNode struct { } // Create implements EventHandler -func (e *EnqueueNodePoolForNode) Create(evt event.CreateEvent, +func (e *EnqueueNodePoolForNode) Create(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) { node, ok := evt.Object.(*corev1.Node) if !ok { @@ -55,7 +56,7 @@ func (e *EnqueueNodePoolForNode) Create(evt event.CreateEvent, } // Update implements EventHandler -func (e *EnqueueNodePoolForNode) Update(evt event.UpdateEvent, +func (e *EnqueueNodePoolForNode) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) { newNode, ok := evt.ObjectNew.(*corev1.Node) if !ok { @@ -110,7 +111,7 @@ func (e *EnqueueNodePoolForNode) Update(evt event.UpdateEvent, } // Delete implements EventHandler -func (e *EnqueueNodePoolForNode) Delete(evt event.DeleteEvent, +func (e *EnqueueNodePoolForNode) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) { node, ok := evt.Object.(*corev1.Node) if !ok { @@ -130,7 +131,7 @@ func (e *EnqueueNodePoolForNode) Delete(evt event.DeleteEvent, } // Generic implements EventHandler -func (e *EnqueueNodePoolForNode) Generic(evt event.GenericEvent, +func (e *EnqueueNodePoolForNode) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) { } diff --git a/pkg/yurtmanager/controller/nodepool/nodepool_enqueue_handlers_test.go b/pkg/yurtmanager/controller/nodepool/nodepool_enqueue_handlers_test.go index 90cabe4753e..3e2ada559c5 100644 --- a/pkg/yurtmanager/controller/nodepool/nodepool_enqueue_handlers_test.go +++ b/pkg/yurtmanager/controller/nodepool/nodepool_enqueue_handlers_test.go @@ -17,6 +17,7 @@ limitations under the License. package nodepool import ( + "context" "testing" corev1 "k8s.io/api/core/v1" @@ -63,7 +64,7 @@ func TestCreate(t *testing.T) { t.Run(k, func(t *testing.T) { handler := &EnqueueNodePoolForNode{} q := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) - handler.Create(tc.event, q) + handler.Create(context.Background(), tc.event, q) if q.Len() != tc.wantedNum { t.Errorf("Expected %d, got %d", tc.wantedNum, q.Len()) @@ -265,7 +266,7 @@ func TestUpdate(t *testing.T) { Recorder: record.NewFakeRecorder(100), } q := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) - handler.Update(tc.event, q) + handler.Update(context.Background(), tc.event, q) if q.Len() != tc.wantedNum { t.Errorf("Expected %d, got %d", tc.wantedNum, q.Len()) @@ -309,7 +310,7 @@ func TestDelete(t *testing.T) { t.Run(k, func(t *testing.T) { handler := &EnqueueNodePoolForNode{} q := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) - handler.Delete(tc.event, q) + handler.Delete(context.Background(), tc.event, q) if q.Len() != tc.wantedNum { t.Errorf("Expected %d, got %d", tc.wantedNum, q.Len()) diff --git a/pkg/yurtmanager/controller/platformadmin/config/config.go b/pkg/yurtmanager/controller/platformadmin/config/config.go index 378082d0b7c..0a72dd1cf2d 100644 --- a/pkg/yurtmanager/controller/platformadmin/config/config.go +++ b/pkg/yurtmanager/controller/platformadmin/config/config.go @@ -65,16 +65,16 @@ type ManifestVersion struct { RequiredComponents []string `yaml:"requiredComponents"` } -func ExtractVersionsName(manifest *Manifest) sets.String { - versionsNameSet := sets.NewString() +func ExtractVersionsName(manifest *Manifest) sets.Set[string] { + versionsNameSet := sets.New[string]() for _, version := range manifest.Versions { versionsNameSet.Insert(version.Name) } return versionsNameSet } -func ExtractRequiredComponentsName(manifest *Manifest, versionName string) sets.String { - requiredComponentSet := sets.NewString() +func ExtractRequiredComponentsName(manifest *Manifest, versionName string) sets.Set[string] { + requiredComponentSet := sets.New[string]() for _, version := range manifest.Versions { if version.Name == versionName { for _, c := range version.RequiredComponents { diff --git a/pkg/yurtmanager/controller/platformadmin/iotdock.go b/pkg/yurtmanager/controller/platformadmin/iotdock.go index bec617f10cf..8894a637240 100644 --- a/pkg/yurtmanager/controller/platformadmin/iotdock.go +++ b/pkg/yurtmanager/controller/platformadmin/iotdock.go @@ -81,7 +81,7 @@ func newYurtIoTDockComponent(platformAdmin *iotv1alpha2.PlatformAdmin, platformA LivenessProbe: &corev1.Probe{ InitialDelaySeconds: 15, PeriodSeconds: 20, - Handler: corev1.Handler{ + ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/healthz", Port: intstr.FromInt(8081), @@ -91,7 +91,7 @@ func newYurtIoTDockComponent(platformAdmin *iotv1alpha2.PlatformAdmin, platformA ReadinessProbe: &corev1.Probe{ InitialDelaySeconds: 5, PeriodSeconds: 10, - Handler: corev1.Handler{ + ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/readyz", Port: intstr.FromInt(8081), diff --git a/pkg/yurtmanager/controller/platformadmin/platformadmin_controller.go b/pkg/yurtmanager/controller/platformadmin/platformadmin_controller.go index fbec9a2d8cb..a7a0567ebe0 100644 --- a/pkg/yurtmanager/controller/platformadmin/platformadmin_controller.go +++ b/pkg/yurtmanager/controller/platformadmin/platformadmin_controller.go @@ -145,31 +145,25 @@ func add(mgr manager.Manager, cfg *appconfig.CompletedConfig, r reconcile.Reconc } // Watch for changes to PlatformAdmin - err = c.Watch(&source.Kind{Type: &iotv1alpha2.PlatformAdmin{}}, &handler.EnqueueRequestForObject{}) + err = c.Watch(source.Kind(mgr.GetCache(), &iotv1alpha2.PlatformAdmin{}), &handler.EnqueueRequestForObject{}) if err != nil { return err } - err = c.Watch(&source.Kind{Type: &corev1.ConfigMap{}}, &handler.EnqueueRequestForOwner{ - IsController: false, - OwnerType: &iotv1alpha2.PlatformAdmin{}, - }) + err = c.Watch(source.Kind(mgr.GetCache(), &corev1.ConfigMap{}), + handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &iotv1alpha2.PlatformAdmin{}, handler.OnlyControllerOwner())) if err != nil { return err } - err = c.Watch(&source.Kind{Type: &corev1.Service{}}, &handler.EnqueueRequestForOwner{ - IsController: false, - OwnerType: &iotv1alpha2.PlatformAdmin{}, - }) + err = c.Watch(source.Kind(mgr.GetCache(), &corev1.Service{}), + handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &iotv1alpha2.PlatformAdmin{}, handler.OnlyControllerOwner())) if err != nil { return err } - err = c.Watch(&source.Kind{Type: &appsv1alpha1.YurtAppSet{}}, &handler.EnqueueRequestForOwner{ - IsController: false, - OwnerType: &iotv1alpha2.PlatformAdmin{}, - }) + err = c.Watch(source.Kind(mgr.GetCache(), &appsv1alpha1.YurtAppSet{}), + handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &iotv1alpha2.PlatformAdmin{}, handler.OnlyControllerOwner())) if err != nil { return err } @@ -455,7 +449,7 @@ func (r *ReconcilePlatformAdmin) reconcileComponent(ctx context.Context, platfor } pool := appsv1alpha1.Pool{ Name: platformAdmin.Spec.PoolName, - Replicas: pointer.Int32Ptr(1), + Replicas: pointer.Int32(1), } pool.NodeSelectorTerm.MatchExpressions = append(pool.NodeSelectorTerm.MatchExpressions, corev1.NodeSelectorRequirement{ @@ -573,7 +567,7 @@ func (r *ReconcilePlatformAdmin) handleYurtAppSet(ctx context.Context, platformA yas.Labels[iotv1alpha2.LabelPlatformAdminGenerate] = LabelDeployment pool := appsv1alpha1.Pool{ Name: platformAdmin.Spec.PoolName, - Replicas: pointer.Int32Ptr(1), + Replicas: pointer.Int32(1), } pool.NodeSelectorTerm.MatchExpressions = append(pool.NodeSelectorTerm.MatchExpressions, corev1.NodeSelectorRequirement{ @@ -837,7 +831,7 @@ func (r *ReconcilePlatformAdmin) calculateDesiredComponents(platformAdmin *iotv1 // Calculate all the components that need to be added or removed and determine whether need to rewrite the framework addedComponentSet := sets.NewString() - for _, componentName := range requiredComponentSet.List() { + for _, componentName := range requiredComponentSet.UnsortedList() { if !frameworkComponentSet.Has(componentName) { addedComponentSet.Insert(componentName) needWriteFramework = true diff --git a/pkg/yurtmanager/controller/raven/dns/dns_controller.go b/pkg/yurtmanager/controller/raven/dns/dns_controller.go index 0ab7edde1d0..661d986329c 100644 --- a/pkg/yurtmanager/controller/raven/dns/dns_controller.go +++ b/pkg/yurtmanager/controller/raven/dns/dns_controller.go @@ -83,7 +83,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { } // Watch for changes to service - err = c.Watch(&source.Kind{Type: &corev1.Service{}}, &EnqueueRequestForServiceEvent{}, predicate.NewPredicateFuncs( + err = c.Watch(source.Kind(mgr.GetCache(), &corev1.Service{}), &EnqueueRequestForServiceEvent{}, predicate.NewPredicateFuncs( func(obj client.Object) bool { svc, ok := obj.(*corev1.Service) if !ok { @@ -98,7 +98,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { return err } //Watch for changes to nodes - err = c.Watch(&source.Kind{Type: &corev1.Node{}}, &EnqueueRequestForNodeEvent{}) + err = c.Watch(source.Kind(mgr.GetCache(), &corev1.Node{}), &EnqueueRequestForNodeEvent{}) if err != nil { return err } @@ -159,7 +159,7 @@ func (r *ReconcileDns) Reconcile(ctx context.Context, req reconcile.Request) (re func (r ReconcileDns) getProxyDNS(ctx context.Context, objKey client.ObjectKey) (*corev1.ConfigMap, error) { var cm corev1.ConfigMap - waitErr := wait.PollImmediate(5*time.Second, time.Minute, func() (done bool, err error) { + waitErr := wait.PollUntilContextTimeout(ctx, 5*time.Second, time.Minute, true, func(ctx context.Context) (done bool, err error) { err = r.Client.Get(ctx, objKey, &cm) if err != nil { if apierrors.IsNotFound(err) { diff --git a/pkg/yurtmanager/controller/raven/dns/dns_enqueue_handlers.go b/pkg/yurtmanager/controller/raven/dns/dns_enqueue_handlers.go index 3faedc77cfe..40009c945d9 100644 --- a/pkg/yurtmanager/controller/raven/dns/dns_enqueue_handlers.go +++ b/pkg/yurtmanager/controller/raven/dns/dns_enqueue_handlers.go @@ -17,6 +17,8 @@ limitations under the License. package dns import ( + "context" + corev1 "k8s.io/api/core/v1" "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" @@ -27,7 +29,7 @@ import ( type EnqueueRequestForServiceEvent struct{} -func (h *EnqueueRequestForServiceEvent) Create(e event.CreateEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForServiceEvent) Create(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { svc, ok := e.Object.(*corev1.Service) if !ok { klog.Error(Format("could not assert runtime Object to v1.Service")) @@ -42,7 +44,7 @@ func (h *EnqueueRequestForServiceEvent) Create(e event.CreateEvent, q workqueue. util.AddDNSConfigmapToWorkQueue(q) } -func (h *EnqueueRequestForServiceEvent) Update(e event.UpdateEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForServiceEvent) Update(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { newSvc, ok := e.ObjectNew.(*corev1.Service) if !ok { klog.Error(Format("could not assert runtime Object to v1.Service")) @@ -59,7 +61,7 @@ func (h *EnqueueRequestForServiceEvent) Update(e event.UpdateEvent, q workqueue. } } -func (h *EnqueueRequestForServiceEvent) Delete(e event.DeleteEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForServiceEvent) Delete(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { _, ok := e.Object.(*corev1.Service) if !ok { klog.Error(Format("could not assert runtime Object to v1.Service")) @@ -70,13 +72,13 @@ func (h *EnqueueRequestForServiceEvent) Delete(e event.DeleteEvent, q workqueue. return } -func (h *EnqueueRequestForServiceEvent) Generic(e event.GenericEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForServiceEvent) Generic(ctx context.Context, e event.GenericEvent, q workqueue.RateLimitingInterface) { return } type EnqueueRequestForNodeEvent struct{} -func (h *EnqueueRequestForNodeEvent) Create(e event.CreateEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForNodeEvent) Create(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { _, ok := e.Object.(*corev1.Node) if !ok { klog.Error(Format("could not assert runtime Object to v1.Node")) @@ -86,11 +88,11 @@ func (h *EnqueueRequestForNodeEvent) Create(e event.CreateEvent, q workqueue.Rat util.AddDNSConfigmapToWorkQueue(q) } -func (h *EnqueueRequestForNodeEvent) Update(e event.UpdateEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForNodeEvent) Update(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { return } -func (h *EnqueueRequestForNodeEvent) Delete(e event.DeleteEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForNodeEvent) Delete(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { _, ok := e.Object.(*corev1.Node) if !ok { klog.Error(Format("could not assert runtime Object to v1.Node")) @@ -100,6 +102,6 @@ func (h *EnqueueRequestForNodeEvent) Delete(e event.DeleteEvent, q workqueue.Rat util.AddDNSConfigmapToWorkQueue(q) } -func (h *EnqueueRequestForNodeEvent) Generic(e event.GenericEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForNodeEvent) Generic(ctx context.Context, e event.GenericEvent, q workqueue.RateLimitingInterface) { } diff --git a/pkg/yurtmanager/controller/raven/dns/dns_enqueue_handlers_test.go b/pkg/yurtmanager/controller/raven/dns/dns_enqueue_handlers_test.go index bb6ff14a945..975eca8b33b 100644 --- a/pkg/yurtmanager/controller/raven/dns/dns_enqueue_handlers_test.go +++ b/pkg/yurtmanager/controller/raven/dns/dns_enqueue_handlers_test.go @@ -17,6 +17,7 @@ limitations under the License. package dns import ( + "context" "testing" "github.com/stretchr/testify/assert" @@ -67,7 +68,7 @@ func TestEnqueueRequestFoServiceEvent(t *testing.T) { queue.Done(item) } } - h.Create(event.CreateEvent{Object: svc}, queue) + h.Create(context.Background(), event.CreateEvent{Object: svc}, queue) if !assert.Equal(t, 1, queue.Len()) { t.Errorf("failed to update service, expected %d, but get %d", 1, queue.Len()) } @@ -76,7 +77,7 @@ func TestEnqueueRequestFoServiceEvent(t *testing.T) { deletedSvc := svc.DeepCopy() time := metav1.Now() deletedSvc.DeletionTimestamp = &time - h.Delete(event.DeleteEvent{Object: deletedSvc}, queue) + h.Delete(context.Background(), event.DeleteEvent{Object: deletedSvc}, queue) if !assert.Equal(t, 1, queue.Len()) { t.Errorf("failed to update service, expected %d, but get %d", 1, queue.Len()) } @@ -84,7 +85,7 @@ func TestEnqueueRequestFoServiceEvent(t *testing.T) { newSvc := svc.DeepCopy() newSvc.Spec.ClusterIP = "0.0.0.0" - h.Update(event.UpdateEvent{ObjectOld: svc, ObjectNew: newSvc}, queue) + h.Update(context.Background(), event.UpdateEvent{ObjectOld: svc, ObjectNew: newSvc}, queue) if !assert.Equal(t, 1, queue.Len()) { t.Errorf("failed to update service, expected %d, but get %d", 1, queue.Len()) } @@ -101,7 +102,7 @@ func TestEnqueueRequestForNodeEvent(t *testing.T) { queue.Done(item) } } - h.Create(event.CreateEvent{Object: node}, queue) + h.Create(context.Background(), event.CreateEvent{Object: node}, queue) if !assert.Equal(t, 1, queue.Len()) { t.Errorf("failed to create node, expected %d, but get %d", 1, queue.Len()) } @@ -110,7 +111,7 @@ func TestEnqueueRequestForNodeEvent(t *testing.T) { time := metav1.Now() deletedNode := node.DeepCopy() deletedNode.DeletionTimestamp = &time - h.Delete(event.DeleteEvent{Object: deletedNode}, queue) + h.Delete(context.Background(), event.DeleteEvent{Object: deletedNode}, queue) if !assert.Equal(t, 1, queue.Len()) { t.Errorf("failed to create node, expected %d, but get %d", 1, queue.Len()) } diff --git a/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_controller.go b/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_controller.go index ac1ef010fc7..694bb3721de 100644 --- a/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_controller.go +++ b/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_controller.go @@ -93,13 +93,13 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { } // Watch for changes to Gateway - err = c.Watch(&source.Kind{Type: &ravenv1beta1.Gateway{}}, &EnqueueRequestForGatewayEvent{}) + err = c.Watch(source.Kind(mgr.GetCache(), &ravenv1beta1.Gateway{}), &EnqueueRequestForGatewayEvent{}) if err != nil { return err } //Watch for changes to raven agent - err = c.Watch(&source.Kind{Type: &corev1.ConfigMap{}}, &EnqueueRequestForConfigEvent{}, predicate.NewPredicateFuncs( + err = c.Watch(source.Kind(mgr.GetCache(), &corev1.ConfigMap{}), &EnqueueRequestForConfigEvent{}, predicate.NewPredicateFuncs( func(object client.Object) bool { cm, ok := object.(*corev1.ConfigMap) if !ok { @@ -383,7 +383,7 @@ func (r *ReconcileService) ensureSpecEndpoints(ctx context.Context, gateways []* func (r *ReconcileService) waitElectEndpoints(ctx context.Context, gwName string) (*ravenv1beta1.Gateway, error) { var gw ravenv1beta1.Gateway - err := wait.PollImmediate(time.Second*5, time.Minute, func() (done bool, err error) { + err := wait.PollUntilContextTimeout(ctx, time.Second*5, time.Minute, true, func(ctx context.Context) (done bool, err error) { err = r.Get(ctx, types.NamespacedName{Name: gwName}, &gw) if err != nil { return false, err diff --git a/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_enqueue_handlers.go b/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_enqueue_handlers.go index 4e473502e88..c5980c035de 100644 --- a/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_enqueue_handlers.go +++ b/pkg/yurtmanager/controller/raven/gatewayinternalservice/gateway_internal_service_enqueue_handlers.go @@ -17,6 +17,7 @@ limitations under the License. package gatewayinternalservice import ( + "context" "net" corev1 "k8s.io/api/core/v1" @@ -30,7 +31,7 @@ import ( type EnqueueRequestForGatewayEvent struct{} -func (h *EnqueueRequestForGatewayEvent) Create(e event.CreateEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForGatewayEvent) Create(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { gw, ok := e.Object.(*ravenv1beta1.Gateway) if !ok { klog.Error(Format("could not assert runtime Object %s/%s to v1beta1.Gateway", e.Object.GetNamespace(), e.Object.GetName())) @@ -43,7 +44,7 @@ func (h *EnqueueRequestForGatewayEvent) Create(e event.CreateEvent, q workqueue. util.AddGatewayProxyInternalService(q) } -func (h *EnqueueRequestForGatewayEvent) Update(e event.UpdateEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForGatewayEvent) Update(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { newGw, ok := e.ObjectNew.(*ravenv1beta1.Gateway) if !ok { klog.Error(Format("could not assert runtime Object %s/%s to v1beta1.Gateway", e.ObjectNew.GetNamespace(), e.ObjectNew.GetName())) @@ -61,7 +62,7 @@ func (h *EnqueueRequestForGatewayEvent) Update(e event.UpdateEvent, q workqueue. util.AddGatewayProxyInternalService(q) } -func (h *EnqueueRequestForGatewayEvent) Delete(e event.DeleteEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForGatewayEvent) Delete(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { gw, ok := e.Object.(*ravenv1beta1.Gateway) if !ok { klog.Error(Format("could not assert runtime Object %s/%s to v1beta1.Gateway", e.Object.GetNamespace(), e.Object.GetName())) @@ -74,13 +75,13 @@ func (h *EnqueueRequestForGatewayEvent) Delete(e event.DeleteEvent, q workqueue. util.AddGatewayProxyInternalService(q) } -func (h *EnqueueRequestForGatewayEvent) Generic(e event.GenericEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForGatewayEvent) Generic(ctx context.Context, e event.GenericEvent, q workqueue.RateLimitingInterface) { return } type EnqueueRequestForConfigEvent struct{} -func (h *EnqueueRequestForConfigEvent) Create(e event.CreateEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForConfigEvent) Create(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { cm, ok := e.Object.(*corev1.ConfigMap) if !ok { klog.Error(Format("could not assert runtime Object %s/%s to v1.Configmap", e.Object.GetNamespace(), e.Object.GetName())) @@ -105,7 +106,7 @@ func (h *EnqueueRequestForConfigEvent) Create(e event.CreateEvent, q workqueue.R } } -func (h *EnqueueRequestForConfigEvent) Update(e event.UpdateEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForConfigEvent) Update(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { newCm, ok := e.ObjectNew.(*corev1.ConfigMap) if !ok { klog.Error(Format("could not assert runtime Object %s/%s to v1.Configmap", e.ObjectNew.GetNamespace(), e.ObjectNew.GetName())) @@ -138,10 +139,10 @@ func (h *EnqueueRequestForConfigEvent) Update(e event.UpdateEvent, q workqueue.R } } -func (h *EnqueueRequestForConfigEvent) Delete(e event.DeleteEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForConfigEvent) Delete(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { return } -func (h *EnqueueRequestForConfigEvent) Generic(e event.GenericEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForConfigEvent) Generic(ctx context.Context, e event.GenericEvent, q workqueue.RateLimitingInterface) { return } diff --git a/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go index 2f7e7065f45..91b9189e9fc 100644 --- a/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go +++ b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go @@ -106,18 +106,18 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { } // Watch for changes to Gateway - err = c.Watch(&source.Kind{Type: &ravenv1beta1.Gateway{}}, &handler.EnqueueRequestForObject{}) + err = c.Watch(source.Kind(mgr.GetCache(), &ravenv1beta1.Gateway{}), &handler.EnqueueRequestForObject{}) if err != nil { return err } // Watch for changes to Nodes - err = c.Watch(&source.Kind{Type: &corev1.Node{}}, &EnqueueGatewayForNode{}) + err = c.Watch(source.Kind(mgr.GetCache(), &corev1.Node{}), &EnqueueGatewayForNode{}) if err != nil { return err } - err = c.Watch(&source.Kind{Type: &corev1.ConfigMap{}}, &EnqueueGatewayForRavenConfig{client: mgr.GetClient()}, predicate.NewPredicateFuncs( + err = c.Watch(source.Kind(mgr.GetCache(), &corev1.ConfigMap{}), &EnqueueGatewayForRavenConfig{client: mgr.GetClient()}, predicate.NewPredicateFuncs( func(object client.Object) bool { cm, ok := object.(*corev1.ConfigMap) if !ok { diff --git a/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_enqueue_handlers.go b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_enqueue_handlers.go index b092285a992..d5e3ce431fc 100644 --- a/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_enqueue_handlers.go +++ b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_enqueue_handlers.go @@ -33,7 +33,7 @@ import ( type EnqueueGatewayForNode struct{} // Create implements EventHandler -func (e *EnqueueGatewayForNode) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) { +func (e *EnqueueGatewayForNode) Create(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) { node, ok := evt.Object.(*corev1.Node) if !ok { klog.Error(Format("could not assert runtime Object to v1.Node")) @@ -49,7 +49,7 @@ func (e *EnqueueGatewayForNode) Create(evt event.CreateEvent, q workqueue.RateLi } // Update implements EventHandler -func (e *EnqueueGatewayForNode) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) { +func (e *EnqueueGatewayForNode) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) { newNode, ok := evt.ObjectNew.(*corev1.Node) if !ok { klog.Errorf(Format("could not assert runtime Object(%s) to v1.Node", @@ -80,7 +80,7 @@ func (e *EnqueueGatewayForNode) Update(evt event.UpdateEvent, q workqueue.RateLi } // Delete implements EventHandler -func (e *EnqueueGatewayForNode) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) { +func (e *EnqueueGatewayForNode) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) { node, ok := evt.Object.(*corev1.Node) if !ok { klog.Error(Format("could not assert runtime Object to v1.Node")) @@ -99,14 +99,14 @@ func (e *EnqueueGatewayForNode) Delete(evt event.DeleteEvent, q workqueue.RateLi } // Generic implements EventHandler -func (e *EnqueueGatewayForNode) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) { +func (e *EnqueueGatewayForNode) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) { } type EnqueueGatewayForRavenConfig struct { client client.Client } -func (e *EnqueueGatewayForRavenConfig) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) { +func (e *EnqueueGatewayForRavenConfig) Create(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) { _, ok := evt.Object.(*corev1.ConfigMap) if !ok { klog.Error(Format("could not assert runtime Object to v1.ConfigMap")) @@ -119,7 +119,7 @@ func (e *EnqueueGatewayForRavenConfig) Create(evt event.CreateEvent, q workqueue } } -func (e *EnqueueGatewayForRavenConfig) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) { +func (e *EnqueueGatewayForRavenConfig) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) { oldCm, ok := evt.ObjectOld.(*corev1.ConfigMap) if !ok { klog.Error(Format("could not assert runtime Object to v1.ConfigMap")) @@ -149,7 +149,7 @@ func (e *EnqueueGatewayForRavenConfig) Update(evt event.UpdateEvent, q workqueue } } -func (e *EnqueueGatewayForRavenConfig) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) { +func (e *EnqueueGatewayForRavenConfig) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) { _, ok := evt.Object.(*corev1.ConfigMap) if !ok { klog.Error(Format("could not assert runtime Object to v1.ConfigMap")) @@ -162,7 +162,7 @@ func (e *EnqueueGatewayForRavenConfig) Delete(evt event.DeleteEvent, q workqueue } } -func (e *EnqueueGatewayForRavenConfig) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) { +func (e *EnqueueGatewayForRavenConfig) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) { } diff --git a/pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_controller.go b/pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_controller.go index 28ed6b81820..a67a3a4aa7a 100644 --- a/pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_controller.go +++ b/pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_controller.go @@ -105,13 +105,13 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { } // Watch for changes to Gateway - err = c.Watch(&source.Kind{Type: &ravenv1beta1.Gateway{}}, &EnqueueRequestForGatewayEvent{}) + err = c.Watch(source.Kind(mgr.GetCache(), &ravenv1beta1.Gateway{}), &EnqueueRequestForGatewayEvent{}) if err != nil { return err } //Watch for changes to raven agent - err = c.Watch(&source.Kind{Type: &corev1.ConfigMap{}}, &EnqueueRequestForConfigEvent{client: mgr.GetClient()}, predicate.NewPredicateFuncs( + err = c.Watch(source.Kind(mgr.GetCache(), &corev1.ConfigMap{}), &EnqueueRequestForConfigEvent{client: mgr.GetClient()}, predicate.NewPredicateFuncs( func(object client.Object) bool { cm, ok := object.(*corev1.ConfigMap) if !ok { diff --git a/pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_enqueue_handlers.go b/pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_enqueue_handlers.go index bce68bb8645..f94f4663dd3 100644 --- a/pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_enqueue_handlers.go +++ b/pkg/yurtmanager/controller/raven/gatewaypublicservice/gateway_public_service_enqueue_handlers.go @@ -32,7 +32,7 @@ import ( type EnqueueRequestForGatewayEvent struct{} -func (h *EnqueueRequestForGatewayEvent) Create(e event.CreateEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForGatewayEvent) Create(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { gw, ok := e.Object.(*ravenv1beta1.Gateway) if !ok { klog.Error(Format("could not assert runtime Object %s/%s to v1beta1.Gateway,", e.Object.GetNamespace(), e.Object.GetName())) @@ -45,7 +45,7 @@ func (h *EnqueueRequestForGatewayEvent) Create(e event.CreateEvent, q workqueue. util.AddGatewayToWorkQueue(gw.GetName(), q) } -func (h *EnqueueRequestForGatewayEvent) Update(e event.UpdateEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForGatewayEvent) Update(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { newGw, ok := e.ObjectNew.(*ravenv1beta1.Gateway) if !ok { klog.Error(Format("could not assert runtime Object %s/%s to v1beta1.Gateway,", e.ObjectNew.GetNamespace(), e.ObjectNew.GetName())) @@ -62,7 +62,7 @@ func (h *EnqueueRequestForGatewayEvent) Update(e event.UpdateEvent, q workqueue. } } -func (h *EnqueueRequestForGatewayEvent) Delete(e event.DeleteEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForGatewayEvent) Delete(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { gw, ok := e.Object.(*ravenv1beta1.Gateway) if !ok { klog.Error(Format("could not assert runtime Object %s/%s to v1beta1.Gateway,", e.Object.GetNamespace(), e.Object.GetName())) @@ -75,7 +75,7 @@ func (h *EnqueueRequestForGatewayEvent) Delete(e event.DeleteEvent, q workqueue. util.AddGatewayToWorkQueue(gw.GetName(), q) } -func (h *EnqueueRequestForGatewayEvent) Generic(e event.GenericEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForGatewayEvent) Generic(ctx context.Context, e event.GenericEvent, q workqueue.RateLimitingInterface) { return } @@ -95,7 +95,7 @@ type EnqueueRequestForConfigEvent struct { client client.Client } -func (h *EnqueueRequestForConfigEvent) Create(e event.CreateEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForConfigEvent) Create(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { cm, ok := e.Object.(*corev1.ConfigMap) if !ok { klog.Error(Format("could not assert runtime Object %s/%s to v1.Configmap,", e.Object.GetNamespace(), e.Object.GetName())) @@ -114,7 +114,7 @@ func (h *EnqueueRequestForConfigEvent) Create(e event.CreateEvent, q workqueue.R } } -func (h *EnqueueRequestForConfigEvent) Update(e event.UpdateEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForConfigEvent) Update(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { newCm, ok := e.ObjectNew.(*corev1.ConfigMap) if !ok { klog.Error(Format("could not assert runtime Object %s/%s to v1.Configmap,", e.ObjectNew.GetNamespace(), e.ObjectNew.GetName())) @@ -139,11 +139,11 @@ func (h *EnqueueRequestForConfigEvent) Update(e event.UpdateEvent, q workqueue.R } } -func (h *EnqueueRequestForConfigEvent) Delete(e event.DeleteEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForConfigEvent) Delete(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { return } -func (h *EnqueueRequestForConfigEvent) Generic(e event.GenericEvent, q workqueue.RateLimitingInterface) { +func (h *EnqueueRequestForConfigEvent) Generic(ctx context.Context, e event.GenericEvent, q workqueue.RateLimitingInterface) { return } diff --git a/pkg/yurtmanager/controller/servicetopology/endpoints/endpoints_controller.go b/pkg/yurtmanager/controller/servicetopology/endpoints/endpoints_controller.go index 6be8d3b05fb..2aeb6e16262 100644 --- a/pkg/yurtmanager/controller/servicetopology/endpoints/endpoints_controller.go +++ b/pkg/yurtmanager/controller/servicetopology/endpoints/endpoints_controller.go @@ -60,13 +60,10 @@ type ReconcileServicetopologyEndpoints struct { // newReconciler returns a new reconcile.Reconciler func newReconciler(_ *appconfig.CompletedConfig, mgr manager.Manager) reconcile.Reconciler { - return &ReconcileServicetopologyEndpoints{} -} - -func (r *ReconcileServicetopologyEndpoints) InjectClient(c client.Client) error { - r.Client = c - r.endpointsAdapter = adapter.NewEndpointsAdapter(c) - return nil + return &ReconcileServicetopologyEndpoints{ + Client: mgr.GetClient(), + endpointsAdapter: adapter.NewEndpointsAdapter(mgr.GetClient()), + } } // add adds a new Controller to mgr with r as the reconcile.Reconciler @@ -78,7 +75,7 @@ func add(mgr manager.Manager, cfg *appconfig.CompletedConfig, r reconcile.Reconc } // Watch for changes to Service - if err := c.Watch(&source.Kind{Type: &corev1.Service{}}, &EnqueueEndpointsForService{ + if err := c.Watch(source.Kind(mgr.GetCache(), &corev1.Service{}), &EnqueueEndpointsForService{ endpointsAdapter: r.(*ReconcileServicetopologyEndpoints).endpointsAdapter, }); err != nil { return err diff --git a/pkg/yurtmanager/controller/servicetopology/endpoints/endpoints_enqueue_handlers.go b/pkg/yurtmanager/controller/servicetopology/endpoints/endpoints_enqueue_handlers.go index b172f5a713b..a2611fac536 100644 --- a/pkg/yurtmanager/controller/servicetopology/endpoints/endpoints_enqueue_handlers.go +++ b/pkg/yurtmanager/controller/servicetopology/endpoints/endpoints_enqueue_handlers.go @@ -17,6 +17,8 @@ limitations under the License. package endpoints import ( + "context" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/cache" @@ -34,12 +36,12 @@ type EnqueueEndpointsForService struct { } // Create implements EventHandler -func (e *EnqueueEndpointsForService) Create(evt event.CreateEvent, +func (e *EnqueueEndpointsForService) Create(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) { } // Update implements EventHandler -func (e *EnqueueEndpointsForService) Update(evt event.UpdateEvent, +func (e *EnqueueEndpointsForService) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) { oldSvc, ok := evt.ObjectOld.(*corev1.Service) if !ok { @@ -59,12 +61,12 @@ func (e *EnqueueEndpointsForService) Update(evt event.UpdateEvent, } // Delete implements EventHandler -func (e *EnqueueEndpointsForService) Delete(evt event.DeleteEvent, +func (e *EnqueueEndpointsForService) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) { } // Generic implements EventHandler -func (e *EnqueueEndpointsForService) Generic(evt event.GenericEvent, +func (e *EnqueueEndpointsForService) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) { } diff --git a/pkg/yurtmanager/controller/servicetopology/endpointslice/endpointslice_controller.go b/pkg/yurtmanager/controller/servicetopology/endpointslice/endpointslice_controller.go index a680cc5d0cf..95ead7a8f6a 100644 --- a/pkg/yurtmanager/controller/servicetopology/endpointslice/endpointslice_controller.go +++ b/pkg/yurtmanager/controller/servicetopology/endpointslice/endpointslice_controller.go @@ -24,7 +24,6 @@ import ( corev1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" discoveryv1beta1 "k8s.io/api/discovery/v1beta1" - "k8s.io/apimachinery/pkg/api/meta" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -54,21 +53,16 @@ func Format(format string, args ...interface{}) string { // Add creates a new Servicetopology endpointslice Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller // and Start it when the Manager is Started. -func Add(ctx context.Context, _ *appconfig.CompletedConfig, mgr manager.Manager) error { - r := &ReconcileServiceTopologyEndpointSlice{} - c, err := controller.New(names.ServiceTopologyEndpointSliceController, mgr, controller.Options{Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) +func Add(ctx context.Context, cfg *appconfig.CompletedConfig, mgr manager.Manager) error { + r := newReconciler(cfg, mgr) + c, err := controller.New(names.ServiceTopologyEndpointSliceController, mgr, + controller.Options{Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) if err != nil { return err } - if r.isSupportEndpointslicev1 { - r.endpointsliceAdapter = adapter.NewEndpointsV1Adapter(r.Client) - } else { - r.endpointsliceAdapter = adapter.NewEndpointsV1Beta1Adapter(r.Client) - } - // Watch for changes to Service - if err := c.Watch(&source.Kind{Type: &corev1.Service{}}, &EnqueueEndpointsliceForService{ + if err := c.Watch(source.Kind(mgr.GetCache(), &corev1.Service{}), &EnqueueEndpointsliceForService{ endpointsliceAdapter: r.endpointsliceAdapter, }); err != nil { return err @@ -87,21 +81,21 @@ type ReconcileServiceTopologyEndpointSlice struct { isSupportEndpointslicev1 bool } -func (r *ReconcileServiceTopologyEndpointSlice) InjectMapper(mapper meta.RESTMapper) error { - if gvk, err := mapper.KindFor(v1EndpointSliceGVR); err != nil { +func newReconciler(_ *appconfig.CompletedConfig, mgr manager.Manager) *ReconcileServiceTopologyEndpointSlice { + r := &ReconcileServiceTopologyEndpointSlice{ + Client: mgr.GetClient(), + } + if gvk, err := mgr.GetRESTMapper().KindFor(v1EndpointSliceGVR); err != nil { klog.Errorf("v1.EndpointSlice is not supported, %v", err) + r.endpointsliceAdapter = adapter.NewEndpointsV1Beta1Adapter(r.Client) r.isSupportEndpointslicev1 = false } else { klog.Infof("%s is supported", gvk.String()) + r.endpointsliceAdapter = adapter.NewEndpointsV1Adapter(r.Client) r.isSupportEndpointslicev1 = true } - return nil -} - -func (r *ReconcileServiceTopologyEndpointSlice) InjectClient(c client.Client) error { - r.Client = c - return nil + return r } // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch diff --git a/pkg/yurtmanager/controller/servicetopology/endpointslice/endpointslice_enqueue_handlers.go b/pkg/yurtmanager/controller/servicetopology/endpointslice/endpointslice_enqueue_handlers.go index 9d08f380b85..c97650ef585 100644 --- a/pkg/yurtmanager/controller/servicetopology/endpointslice/endpointslice_enqueue_handlers.go +++ b/pkg/yurtmanager/controller/servicetopology/endpointslice/endpointslice_enqueue_handlers.go @@ -17,6 +17,8 @@ limitations under the License. package endpointslice import ( + "context" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/cache" @@ -34,12 +36,12 @@ type EnqueueEndpointsliceForService struct { } // Create implements EventHandler -func (e *EnqueueEndpointsliceForService) Create(evt event.CreateEvent, +func (e *EnqueueEndpointsliceForService) Create(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) { } // Update implements EventHandler -func (e *EnqueueEndpointsliceForService) Update(evt event.UpdateEvent, +func (e *EnqueueEndpointsliceForService) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) { oldSvc, ok := evt.ObjectOld.(*corev1.Service) if !ok { @@ -59,12 +61,12 @@ func (e *EnqueueEndpointsliceForService) Update(evt event.UpdateEvent, } // Delete implements EventHandler -func (e *EnqueueEndpointsliceForService) Delete(evt event.DeleteEvent, +func (e *EnqueueEndpointsliceForService) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) { } // Generic implements EventHandler -func (e *EnqueueEndpointsliceForService) Generic(evt event.GenericEvent, +func (e *EnqueueEndpointsliceForService) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) { } diff --git a/pkg/yurtmanager/controller/testutil/test_utils.go b/pkg/yurtmanager/controller/testutil/test_utils.go index 8b64b0b4906..1f75998a5f4 100644 --- a/pkg/yurtmanager/controller/testutil/test_utils.go +++ b/pkg/yurtmanager/controller/testutil/test_utils.go @@ -32,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/watch" @@ -60,6 +61,17 @@ type ImprovedFakeNodeHandler struct { UpdatedNodeStatuses []*v1.Node } +func podIndexer(rawObj ctlclient.Object) []string { + pod, ok := rawObj.(*v1.Pod) + if !ok { + return []string{} + } + if len(pod.Spec.NodeName) == 0 { + return []string{} + } + return []string{pod.Spec.NodeName} +} + func NewImprovedFakeNodeHandler(nodes []*v1.Node, pods *v1.PodList) *ImprovedFakeNodeHandler { scheme := runtime.NewScheme() clientgoscheme.AddToScheme(scheme) @@ -70,6 +82,7 @@ func NewImprovedFakeNodeHandler(nodes []*v1.Node, pods *v1.PodList) *ImprovedFak if pods != nil { clientBuilder.WithLists(pods) } + clientBuilder.WithIndex(&v1.Pod{}, "spec.nodeName", podIndexer) delegateClient := clientBuilder.Build() clientWrapper := NewClientWrapper(delegateClient, scheme) m := &ImprovedFakeNodeHandler{ @@ -117,7 +130,7 @@ func (m *ImprovedFakeNodeHandler) UpdateNodeStatuses(updatedNodeStatuses map[str for _, node := range nodeList.Items { if status, ok := updatedNodeStatuses[node.Name]; ok { node.Status = status - if err = m.baseClient.Status().Update(context.TODO(), &node, &ctlclient.UpdateOptions{}); err != nil { + if err = m.baseClient.Status().Update(context.TODO(), &node, &ctlclient.SubResourceUpdateOptions{}); err != nil { return err } } @@ -203,7 +216,8 @@ func NewClientWrapper(client ctlclient.Client, scheme *runtime.Scheme) *ClientWr } } -func (m *ClientWrapper) Get(ctx context.Context, key ctlclient.ObjectKey, obj ctlclient.Object) error { +// func (m *ClientWrapper) Get(ctx context.Context, key ctlclient.ObjectKey, obj ctlclient.Object) error { +func (m *ClientWrapper) Get(ctx context.Context, key ctlclient.ObjectKey, obj ctlclient.Object, opts ...ctlclient.GetOption) error { m.Lock() defer m.Unlock() gvk, err := apiutil.GVKForObject(obj, m.scheme) @@ -337,7 +351,7 @@ type fakeStatusWriter struct { statusWriter ctlclient.StatusWriter } -func (sw *fakeStatusWriter) Update(ctx context.Context, obj ctlclient.Object, opts ...ctlclient.UpdateOption) error { +func (sw *fakeStatusWriter) Update(ctx context.Context, obj ctlclient.Object, opts ...ctlclient.SubResourceUpdateOption) error { // TODO(droot): This results in full update of the obj (spec + status). Need // a way to update status field only. gvk, err := apiutil.GVKForObject(obj, sw.client.scheme) @@ -377,7 +391,7 @@ func (sw *fakeStatusWriter) Update(ctx context.Context, obj ctlclient.Object, op return nil } -func (sw *fakeStatusWriter) Patch(ctx context.Context, obj ctlclient.Object, patch ctlclient.Patch, opts ...ctlclient.PatchOption) error { +func (sw *fakeStatusWriter) Patch(ctx context.Context, obj ctlclient.Object, patch ctlclient.Patch, opts ...ctlclient.SubResourcePatchOption) error { // TODO(droot): This results in full update of the obj (spec + status). Need // a way to update status field only. patchData, err := patch.Data(obj) @@ -415,6 +429,14 @@ func (sw *fakeStatusWriter) Patch(ctx context.Context, obj ctlclient.Object, pat return nil } +func (sw *fakeStatusWriter) Create(ctx context.Context, obj ctlclient.Object, subResource ctlclient.Object, opts ...ctlclient.SubResourceCreateOption) error { + return nil +} + +func (m *ClientWrapper) SubResource(subResource string) ctlclient.SubResourceClient { + return m.delegateClient.SubResource(subResource) +} + func (m *ClientWrapper) Status() ctlclient.StatusWriter { return &fakeStatusWriter{ client: m, @@ -431,6 +453,14 @@ func (m *ClientWrapper) RESTMapper() meta.RESTMapper { return m.delegateClient.RESTMapper() } +func (m *ClientWrapper) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + return m.delegateClient.GroupVersionKindFor(obj) +} + +func (m *ClientWrapper) IsObjectNamespaced(obj runtime.Object) (bool, error) { + return m.delegateClient.IsObjectNamespaced(obj) +} + func (m *ClientWrapper) ClearActions() { m.Lock() defer m.Unlock() @@ -620,7 +650,7 @@ func (m *FakeNodeHandler) UpdateStatus(ctx context.Context, node *v1.Node, _ met m.lock.Unlock() }() - if err := m.runtimeClient.Status().Update(ctx, node, &ctlclient.UpdateOptions{}); err != nil { + if err := m.runtimeClient.Status().Update(ctx, node, &ctlclient.SubResourceUpdateOptions{}); err != nil { return node, err } diff --git a/pkg/yurtmanager/controller/util/node/controller_utils.go b/pkg/yurtmanager/controller/util/node/controller_utils.go index e9a1e92e201..825db699007 100644 --- a/pkg/yurtmanager/controller/util/node/controller_utils.go +++ b/pkg/yurtmanager/controller/util/node/controller_utils.go @@ -164,7 +164,7 @@ func MarkPodsNotReady(ctx context.Context, c client.Client, recorder record.Even klog.V(2).InfoS("Updating ready status of pod to false", "pod", klog.KObj(pod)) //if _, err := kubeClient.CoreV1().Pods(pod.Namespace).UpdateStatus(ctx, pod, metav1.UpdateOptions{}); err != nil { - if err := c.Status().Update(ctx, pod, &client.UpdateOptions{}); err != nil { + if err := c.Status().Update(ctx, pod, &client.SubResourceUpdateOptions{}); err != nil { if apierrors.IsNotFound(err) { // NotFound error means that pod was already deleted. // There is nothing left to do with this pod. diff --git a/pkg/yurtmanager/controller/util/tools.go b/pkg/yurtmanager/controller/util/tools.go index 85a44b46a37..f8095dbd9df 100644 --- a/pkg/yurtmanager/controller/util/tools.go +++ b/pkg/yurtmanager/controller/util/tools.go @@ -81,10 +81,6 @@ func NewNoReconcileController(name string, mgr manager.Manager, options controll return nil, fmt.Errorf("must specify Name for Controller") } - if options.Log == nil { - options.Log = mgr.GetLogger() - } - if options.CacheSyncTimeout == 0 { options.CacheSyncTimeout = 2 * time.Minute } @@ -93,18 +89,12 @@ func NewNoReconcileController(name string, mgr manager.Manager, options controll options.RateLimiter = workqueue.DefaultControllerRateLimiter() } - // Inject dependencies into Reconciler - if err := mgr.SetFields(options.Reconciler); err != nil { - return nil, err - } - // Create controller with dependencies set c := &controllerimpl.Controller{ MakeQueue: func() workqueue.RateLimitingInterface { return workqueue.NewNamedRateLimitingQueue(options.RateLimiter, name) }, CacheSyncTimeout: options.CacheSyncTimeout, - SetFields: mgr.SetFields, Name: name, RecoverPanic: options.RecoverPanic, } diff --git a/pkg/yurtmanager/controller/yurtappdaemon/nodepool_enqueue_handlers.go b/pkg/yurtmanager/controller/yurtappdaemon/nodepool_enqueue_handlers.go index 00496739eea..02944a19642 100644 --- a/pkg/yurtmanager/controller/yurtappdaemon/nodepool_enqueue_handlers.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/nodepool_enqueue_handlers.go @@ -33,19 +33,19 @@ type EnqueueYurtAppDaemonForNodePool struct { client client.Client } -func (e *EnqueueYurtAppDaemonForNodePool) Create(event event.CreateEvent, limitingInterface workqueue.RateLimitingInterface) { +func (e *EnqueueYurtAppDaemonForNodePool) Create(ctx context.Context, event event.CreateEvent, limitingInterface workqueue.RateLimitingInterface) { e.addAllYurtAppDaemonToWorkQueue(limitingInterface) } -func (e *EnqueueYurtAppDaemonForNodePool) Update(event event.UpdateEvent, limitingInterface workqueue.RateLimitingInterface) { +func (e *EnqueueYurtAppDaemonForNodePool) Update(ctx context.Context, event event.UpdateEvent, limitingInterface workqueue.RateLimitingInterface) { e.addAllYurtAppDaemonToWorkQueue(limitingInterface) } -func (e *EnqueueYurtAppDaemonForNodePool) Delete(event event.DeleteEvent, limitingInterface workqueue.RateLimitingInterface) { +func (e *EnqueueYurtAppDaemonForNodePool) Delete(ctx context.Context, event event.DeleteEvent, limitingInterface workqueue.RateLimitingInterface) { e.addAllYurtAppDaemonToWorkQueue(limitingInterface) } -func (e *EnqueueYurtAppDaemonForNodePool) Generic(event event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) { +func (e *EnqueueYurtAppDaemonForNodePool) Generic(ctx context.Context, event event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) { return } diff --git a/pkg/yurtmanager/controller/yurtappdaemon/nodepool_enqueue_handlers_test.go b/pkg/yurtmanager/controller/yurtappdaemon/nodepool_enqueue_handlers_test.go index 0eb99d4c9e1..f4ab2cc8e9e 100644 --- a/pkg/yurtmanager/controller/yurtappdaemon/nodepool_enqueue_handlers_test.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/nodepool_enqueue_handlers_test.go @@ -17,6 +17,7 @@ limitations under the License. package yurtappdaemon import ( + "context" "testing" "time" @@ -109,7 +110,7 @@ func TestCreate(t *testing.T) { t.Parallel() t.Logf("\tTestCase: %s", st.name) { - ep.Create(st.event, st.limitingInterface) + ep.Create(context.Background(), st.event, st.limitingInterface) get := st.expect if get != st.expect { t.Fatalf("\t%s\texpect %v, but get %v", failed, st.expect, get) @@ -171,7 +172,7 @@ func TestUpdate(t *testing.T) { t.Parallel() t.Logf("\tTestCase: %s", st.name) { - ep.Update(st.event, st.limitingInterface) + ep.Update(context.Background(), st.event, st.limitingInterface) get := st.expect if get != st.expect { t.Fatalf("\t%s\texpect %v, but get %v", failed, st.expect, get) @@ -225,7 +226,7 @@ func TestDelete(t *testing.T) { t.Parallel() t.Logf("\tTestCase: %s", st.name) { - ep.Delete(st.event, st.limitingInterface) + ep.Delete(context.Background(), st.event, st.limitingInterface) get := st.expect if get != st.expect { t.Fatalf("\t%s\texpect %v, but get %v", failed, st.expect, get) @@ -278,7 +279,7 @@ func TestGeneric(t *testing.T) { t.Parallel() t.Logf("\tTestCase: %s", st.name) { - ep.Generic(st.event, st.limitingInterface) + ep.Generic(context.Background(), st.event, st.limitingInterface) get := st.expect if get != st.expect { t.Fatalf("\t%s\texpect %v, but get %v", failed, st.expect, get) diff --git a/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller.go b/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller.go index feea609a555..05a95f01e26 100644 --- a/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller.go @@ -85,13 +85,13 @@ func add(mgr manager.Manager, cfg *config.CompletedConfig, r reconcile.Reconcile } // Watch for changes to YurtAppDaemon - err = c.Watch(&source.Kind{Type: &unitv1alpha1.YurtAppDaemon{}}, &handler.EnqueueRequestForObject{}) + err = c.Watch(source.Kind(mgr.GetCache(), &unitv1alpha1.YurtAppDaemon{}), &handler.EnqueueRequestForObject{}) if err != nil { return err } // Watch for changes to NodePool - err = c.Watch(&source.Kind{Type: &unitv1alpha1.NodePool{}}, &EnqueueYurtAppDaemonForNodePool{client: mgr.GetClient()}) + err = c.Watch(source.Kind(mgr.GetCache(), &unitv1alpha1.NodePool{}), &EnqueueYurtAppDaemonForNodePool{client: mgr.GetClient()}) if err != nil { return err } diff --git a/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller.go b/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller.go index 9d9dc09e48a..3bb4bdd8b55 100644 --- a/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller.go +++ b/pkg/yurtmanager/controller/yurtappoverrider/yurtappoverrider_controller.go @@ -95,7 +95,7 @@ func add(mgr manager.Manager, cfg *appconfig.CompletedConfig, r reconcile.Reconc } // Watch for changes to YurtAppOverrider - err = c.Watch(&source.Kind{Type: &appsv1alpha1.YurtAppOverrider{}}, &handler.EnqueueRequestForObject{}) + err = c.Watch(source.Kind(mgr.GetCache(), &appsv1alpha1.YurtAppOverrider{}), &handler.EnqueueRequestForObject{}) if err != nil { return err } diff --git a/pkg/yurtmanager/controller/yurtappset/workloadmanager/util.go b/pkg/yurtmanager/controller/yurtappset/workloadmanager/util.go index 638ac774781..d06742fec99 100644 --- a/pkg/yurtmanager/controller/yurtappset/workloadmanager/util.go +++ b/pkg/yurtmanager/controller/yurtappset/workloadmanager/util.go @@ -63,14 +63,14 @@ func NewLabelSelectorForYurtAppSet(yas *v1beta1.YurtAppSet) (*metav1.LabelSelect // Get selecetd NodePools from YurtAppSet // return sets for deduplication of NodePools -func GetNodePoolsFromYurtAppSet(cli client.Client, yas *v1beta1.YurtAppSet) (npNames sets.String, err error) { +func GetNodePoolsFromYurtAppSet(cli client.Client, yas *v1beta1.YurtAppSet) (npNames sets.Set[string], err error) { return getSelectedNodepools(cli, yas.Spec.Pools, yas.Spec.NodePoolSelector) } // Get NodePools selected by pools and npSelector // If specified pool does not exist, it will skip -func getSelectedNodepools(cli client.Client, pools []string, npSelector *metav1.LabelSelector) (selectedNps sets.String, err error) { - selectedNps = sets.NewString() +func getSelectedNodepools(cli client.Client, pools []string, npSelector *metav1.LabelSelector) (selectedNps sets.Set[string], err error) { + selectedNps = sets.New[string]() // get all nodepools allNps := v1beta1.NodePoolList{} diff --git a/pkg/yurtmanager/controller/yurtappset/workloadmanager/util_test.go b/pkg/yurtmanager/controller/yurtappset/workloadmanager/util_test.go index 6778495a301..be095926be1 100644 --- a/pkg/yurtmanager/controller/yurtappset/workloadmanager/util_test.go +++ b/pkg/yurtmanager/controller/yurtappset/workloadmanager/util_test.go @@ -179,8 +179,8 @@ func TestGetNodePoolsFromYurtAppSet(t *testing.T) { t.Errorf("GetNodePoolsFromYurtAppSet() error = %v, wantErr %v", err, tt.wantErr) return } - if !reflect.DeepEqual(gotNps.List(), tt.wantNps) { - t.Errorf("GetNodePoolsFromYurtAppSet() gotNps = %v, want %v", gotNps.List(), tt.wantNps) + if !reflect.DeepEqual(gotNps.UnsortedList(), tt.wantNps) { + t.Errorf("GetNodePoolsFromYurtAppSet() gotNps = %v, want %v", gotNps.UnsortedList(), tt.wantNps) } }) } diff --git a/pkg/yurtmanager/controller/yurtappset/yurtappset_controller.go b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller.go index 50cfe403f88..bd3c762f6de 100644 --- a/pkg/yurtmanager/controller/yurtappset/yurtappset_controller.go +++ b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller.go @@ -143,10 +143,10 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { }, } - nodePoolToYurtAppSet := func(nodePool client.Object) (res []reconcile.Request) { + nodePoolToYurtAppSet := func(ctx context.Context, nodePool client.Object) (res []reconcile.Request) { res = make([]reconcile.Request, 0) yasList := &unitv1beta1.YurtAppSetList{} - if err := mgr.GetClient().List(context.TODO(), yasList); err != nil { + if err := mgr.GetClient().List(ctx, yasList); err != nil { return } @@ -160,20 +160,18 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { return } - err = c.Watch(&source.Kind{Type: &unitv1beta1.NodePool{}}, handler.EnqueueRequestsFromMapFunc(nodePoolToYurtAppSet), nodePoolPredicate) + err = c.Watch(source.Kind(mgr.GetCache(), &unitv1beta1.NodePool{}), handler.EnqueueRequestsFromMapFunc(nodePoolToYurtAppSet), nodePoolPredicate) if err != nil { return err } - err = c.Watch(&source.Kind{Type: &unitv1beta1.YurtAppSet{}}, &handler.EnqueueRequestForObject{}) + err = c.Watch(source.Kind(mgr.GetCache(), &unitv1beta1.YurtAppSet{}), &handler.EnqueueRequestForObject{}) if err != nil { return err } - err = c.Watch(&source.Kind{Type: &appsv1.Deployment{}}, &handler.EnqueueRequestForOwner{ - IsController: true, - OwnerType: &unitv1beta1.YurtAppSet{}, - }) + err = c.Watch(source.Kind(mgr.GetCache(), &appsv1.Deployment{}), + handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &unitv1beta1.YurtAppSet{}, handler.OnlyControllerOwner())) if err != nil { return err } @@ -252,7 +250,7 @@ func (r *ReconcileYurtAppSet) Reconcile(_ context.Context, request reconcile.Req return } -func (r *ReconcileYurtAppSet) getNodePoolsFromYurtAppSet(yas *unitv1beta1.YurtAppSet, newStatus *unitv1beta1.YurtAppSetStatus) (npNames sets.String, err error) { +func (r *ReconcileYurtAppSet) getNodePoolsFromYurtAppSet(yas *unitv1beta1.YurtAppSet, newStatus *unitv1beta1.YurtAppSetStatus) (npNames sets.Set[string], err error) { expectedNps, err := workloadmanager.GetNodePoolsFromYurtAppSet(r.Client, yas) if err != nil { return nil, err @@ -262,8 +260,8 @@ func (r *ReconcileYurtAppSet) getNodePoolsFromYurtAppSet(yas *unitv1beta1.YurtAp r.recorder.Event(yas.DeepCopy(), corev1.EventTypeWarning, fmt.Sprintf("No%s", eventTypeFindPools), fmt.Sprintf("There are no matched nodepools for YurtAppSet %s/%s", yas.Namespace, yas.Name)) SetYurtAppSetCondition(newStatus, NewYurtAppSetCondition(unitv1beta1.AppSetPoolFound, corev1.ConditionFalse, fmt.Sprintf("No%s", eventTypeFindPools), "There are no matched nodepools for YurtAppSet")) } else { - klog.V(4).Infof("NodePools matched for YurtAppSet %s/%s: %v", yas.Namespace, yas.Name, expectedNps.List()) - SetYurtAppSetCondition(newStatus, NewYurtAppSetCondition(unitv1beta1.AppSetPoolFound, corev1.ConditionTrue, eventTypeFindPools, fmt.Sprintf("There are %d matched nodepools: %v", expectedNps.Len(), expectedNps.List()))) + klog.V(4).Infof("NodePools matched for YurtAppSet %s/%s: %v", yas.Namespace, yas.Name, expectedNps.UnsortedList()) + SetYurtAppSetCondition(newStatus, NewYurtAppSetCondition(unitv1beta1.AppSetPoolFound, corev1.ConditionTrue, eventTypeFindPools, fmt.Sprintf("There are %d matched nodepools: %v", expectedNps.Len(), expectedNps.UnsortedList()))) } return expectedNps, nil } @@ -282,7 +280,7 @@ func (r *ReconcileYurtAppSet) getWorkloadManagerFromYurtAppSet(yas *unitv1beta1. } func classifyWorkloads(yas *unitv1beta1.YurtAppSet, currentWorkloads []metav1.Object, - expectedNodePools sets.String, expectedRevision string) (needDeleted, needUpdate []metav1.Object, needCreate []string) { + expectedNodePools sets.Set[string], expectedRevision string) (needDeleted, needUpdate []metav1.Object, needCreate []string) { // classify workloads by nodepool name nodePoolsToWorkloads := make(map[string]metav1.Object) @@ -334,7 +332,7 @@ func classifyWorkloads(yas *unitv1beta1.YurtAppSet, currentWorkloads []metav1.Ob } // Conciliate workloads as yas spec expect -func (r *ReconcileYurtAppSet) conciliateWorkloads(yas *unitv1beta1.YurtAppSet, expectedRevision *appsv1.ControllerRevision, newStatus *unitv1beta1.YurtAppSetStatus) (expectedNps sets.String, curWorkloads []metav1.Object, err error) { +func (r *ReconcileYurtAppSet) conciliateWorkloads(yas *unitv1beta1.YurtAppSet, expectedRevision *appsv1.ControllerRevision, newStatus *unitv1beta1.YurtAppSetStatus) (expectedNps sets.Set[string], curWorkloads []metav1.Object, err error) { // Get yas selected NodePools // this may infect yas poolfound condition @@ -447,7 +445,7 @@ func (r *ReconcileYurtAppSet) conciliateWorkloads(yas *unitv1beta1.YurtAppSet, e return } -func (r *ReconcileYurtAppSet) conciliateYurtAppSet(yas *unitv1beta1.YurtAppSet, curWorkloads []metav1.Object, allRevisions []*apps.ControllerRevision, expectedRevison *appsv1.ControllerRevision, expectedNps sets.String, newStatus *unitv1beta1.YurtAppSetStatus) error { +func (r *ReconcileYurtAppSet) conciliateYurtAppSet(yas *unitv1beta1.YurtAppSet, curWorkloads []metav1.Object, allRevisions []*apps.ControllerRevision, expectedRevison *appsv1.ControllerRevision, expectedNps sets.Set[string], newStatus *unitv1beta1.YurtAppSetStatus) error { if err := r.conciliateYurtAppSetStatus(yas, curWorkloads, expectedRevison, expectedNps, newStatus); err != nil { return err } @@ -455,7 +453,7 @@ func (r *ReconcileYurtAppSet) conciliateYurtAppSet(yas *unitv1beta1.YurtAppSet, } // update yas status and clean unused revisions -func (r *ReconcileYurtAppSet) conciliateYurtAppSetStatus(yas *unitv1beta1.YurtAppSet, curWorkloads []metav1.Object, expectedRevison *appsv1.ControllerRevision, expectedNps sets.String, newStatus *unitv1beta1.YurtAppSetStatus) error { +func (r *ReconcileYurtAppSet) conciliateYurtAppSetStatus(yas *unitv1beta1.YurtAppSet, curWorkloads []metav1.Object, expectedRevison *appsv1.ControllerRevision, expectedNps sets.Set[string], newStatus *unitv1beta1.YurtAppSetStatus) error { // calculate yas current status readyWorkloads, updatedWorkloads := 0, 0 diff --git a/pkg/yurtmanager/controller/yurtappset/yurtappset_controller_test.go b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller_test.go index f34bead7a13..c59568af5d5 100644 --- a/pkg/yurtmanager/controller/yurtappset/yurtappset_controller_test.go +++ b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller_test.go @@ -101,7 +101,7 @@ func TestGetWorkloadManagerFromYurtAppSet(t *testing.T) { func TestClassifyWorkloads(t *testing.T) { yas := &v1beta1.YurtAppSet{} - expectedNodePools := sets.NewString("test-np1", "test-np3") + expectedNodePools := sets.New[string]("test-np1", "test-np3") expectedRevision := "test-revision-2" workloadTobeDeleted := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/yurtmanager/controller/yurtcoordinator/cert/certificate.go b/pkg/yurtmanager/controller/yurtcoordinator/cert/certificate.go index dd44020932f..85405701fa1 100644 --- a/pkg/yurtmanager/controller/yurtcoordinator/cert/certificate.go +++ b/pkg/yurtmanager/controller/yurtcoordinator/cert/certificate.go @@ -300,7 +300,7 @@ func GetCertAndKeyFromCertMgr(certManager certificate.Manager, stopCh <-chan str // waiting for the certificate is generated certManager.Start() - err = wait.PollUntil(5*time.Second, func() (bool, error) { + err = wait.PollUntilContextCancel(context.Background(), 5*time.Second, true, func(ctx context.Context) (bool, error) { // keep polling until the certificate is signed if certManager.Current() != nil { klog.Infof(Format("%s certificate signed successfully", ComponentName)) @@ -308,7 +308,7 @@ func GetCertAndKeyFromCertMgr(certManager certificate.Manager, stopCh <-chan str } klog.Infof(Format("waiting for the master to sign the %s certificate", ComponentName)) return false, nil - }, stopCh) + }) if err != nil { return nil, nil, err diff --git a/pkg/yurtmanager/controller/yurtcoordinator/cert/util.go b/pkg/yurtmanager/controller/yurtcoordinator/cert/util.go index c3e61416277..4760cc98ae8 100644 --- a/pkg/yurtmanager/controller/yurtcoordinator/cert/util.go +++ b/pkg/yurtmanager/controller/yurtcoordinator/cert/util.go @@ -56,14 +56,14 @@ func waitUntilSVCReady(clientSet client.Interface, serviceName string, stopCh <- var serverSVC *corev1.Service // wait until get tls server Service - if err = wait.PollUntil(1*time.Second, func() (bool, error) { + if err = wait.PollUntilContextCancel(context.Background(), 1*time.Second, true, func(ctx context.Context) (bool, error) { serverSVC, err = clientSet.CoreV1().Services(YurtCoordinatorNS).Get(context.TODO(), serviceName, metav1.GetOptions{}) if err == nil { klog.Infof(Format("%s service is ready for yurtcoordinator_cert_manager", serviceName)) return true, nil } return false, nil - }, stopCh); err != nil { + }); err != nil { return nil, nil, err } diff --git a/pkg/yurtmanager/controller/yurtcoordinator/cert/yurtcoordinatorcert_controller.go b/pkg/yurtmanager/controller/yurtcoordinator/cert/yurtcoordinatorcert_controller.go index b29d0bcd74c..9d7429050e2 100644 --- a/pkg/yurtmanager/controller/yurtcoordinator/cert/yurtcoordinatorcert_controller.go +++ b/pkg/yurtmanager/controller/yurtcoordinator/cert/yurtcoordinatorcert_controller.go @@ -28,7 +28,6 @@ import ( certificatesv1 "k8s.io/api/certificates/v1" corev1 "k8s.io/api/core/v1" client "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" @@ -204,7 +203,15 @@ func Format(format string, args ...interface{}) string { // Add creates a new YurtCoordinatorcert Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(ctx context.Context, cfg *appconfig.CompletedConfig, mgr manager.Manager) error { - r := &ReconcileYurtCoordinatorCert{} + kubeClient, err := client.NewForConfig(mgr.GetConfig()) + if err != nil { + klog.Errorf("could not create kube client, %v", err) + return err + } + + r := &ReconcileYurtCoordinatorCert{ + kubeClient: kubeClient, + } // Create a new controller c, err := controller.New(names.YurtCoordinatorCertController, mgr, controller.Options{ @@ -264,7 +271,7 @@ func Add(ctx context.Context, cfg *appconfig.CompletedConfig, mgr manager.Manage return false }, } - return c.Watch(&source.Kind{Type: &corev1.Service{}}, &handler.EnqueueRequestForObject{}, svcReadyPredicates) + return c.Watch(source.Kind(mgr.GetCache(), &corev1.Service{}), &handler.EnqueueRequestForObject{}, svcReadyPredicates) } func isYurtCoordinatorSvc(svc *corev1.Service) bool { @@ -289,17 +296,6 @@ type ReconcileYurtCoordinatorCert struct { reuseCA bool } -// InjectConfig will prepare kube client for YurtCoordinatorCert -func (r *ReconcileYurtCoordinatorCert) InjectConfig(cfg *rest.Config) error { - kubeClient, err := client.NewForConfig(cfg) - if err != nil { - klog.Errorf("could not create kube client, %v", err) - return err - } - r.kubeClient = kubeClient - return nil -} - // +kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests,verbs=create // +kubebuilder:rbac:groups="",namespace=kube-system,resources=secrets,verbs=get;update;create;patch // +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;watch;list diff --git a/pkg/yurtmanager/controller/yurtcoordinator/delegatelease/delegatelease_controller.go b/pkg/yurtmanager/controller/yurtcoordinator/delegatelease/delegatelease_controller.go index f0c6153ef32..927088f7313 100644 --- a/pkg/yurtmanager/controller/yurtcoordinator/delegatelease/delegatelease_controller.go +++ b/pkg/yurtmanager/controller/yurtcoordinator/delegatelease/delegatelease_controller.go @@ -25,7 +25,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -51,9 +50,17 @@ type ReconcileDelegateLease struct { // Add creates a delegatelease controller and add it to the Manager with default RBAC. The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(_ context.Context, cfg *appconfig.CompletedConfig, mgr manager.Manager) error { + kubeClient, err := kubernetes.NewForConfig(mgr.GetConfig()) + if err != nil { + klog.Errorf("could not create kube client, %v", err) + return err + } + r := &ReconcileDelegateLease{ - ldc: utils.NewLeaseDelegatedCounter(), - delLdc: utils.NewLeaseDelegatedCounter(), + ldc: utils.NewLeaseDelegatedCounter(), + delLdc: utils.NewLeaseDelegatedCounter(), + Client: mgr.GetClient(), + dlClient: kubeClient, } c, err := controller.New(names.DelegateLeaseController, mgr, controller.Options{ Reconciler: r, MaxConcurrentReconciles: int(cfg.ComponentConfig.DelegateLeaseController.ConcurrentDelegateLeaseWorkers), @@ -61,7 +68,7 @@ func Add(_ context.Context, cfg *appconfig.CompletedConfig, mgr manager.Manager) if err != nil { return err } - err = c.Watch(&source.Kind{Type: &coordv1.Lease{}}, &handler.EnqueueRequestForObject{}) + err = c.Watch(source.Kind(mgr.GetCache(), &coordv1.Lease{}), &handler.EnqueueRequestForObject{}) return err } @@ -196,18 +203,3 @@ func (r *ReconcileDelegateLease) checkNodeReadyConditionAndSetIt(ctx context.Con } klog.Infof("successful set node %s ready condition with true", newNode.Name) } - -func (r *ReconcileDelegateLease) InjectClient(c client.Client) error { - r.Client = c - return nil -} - -func (r *ReconcileDelegateLease) InjectConfig(cfg *rest.Config) error { - client, err := kubernetes.NewForConfig(cfg) - if err != nil { - klog.Errorf("could not create kube client, %v", err) - return err - } - r.dlClient = client - return nil -} diff --git a/pkg/yurtmanager/controller/yurtcoordinator/podbinding/podbinding_controller.go b/pkg/yurtmanager/controller/yurtcoordinator/podbinding/podbinding_controller.go index b5978a1fd69..05b03382de6 100644 --- a/pkg/yurtmanager/controller/yurtcoordinator/podbinding/podbinding_controller.go +++ b/pkg/yurtmanager/controller/yurtcoordinator/podbinding/podbinding_controller.go @@ -71,7 +71,9 @@ func Add(ctx context.Context, c *appconfig.CompletedConfig, mgr manager.Manager) // newReconciler returns a new reconcile.Reconciler func newReconciler(_ *appconfig.CompletedConfig, mgr manager.Manager) reconcile.Reconciler { - return &ReconcilePodBinding{} + return &ReconcilePodBinding{ + Client: mgr.GetClient(), + } } // add adds a new Controller to mgr with r as the reconcile.Reconciler @@ -83,7 +85,7 @@ func add(mgr manager.Manager, cfg *appconfig.CompletedConfig, r reconcile.Reconc return err } - return c.Watch(&source.Kind{Type: &corev1.Node{}}, &handler.EnqueueRequestForObject{}) + return c.Watch(source.Kind(mgr.GetCache(), &corev1.Node{}), &handler.EnqueueRequestForObject{}) //err = c.Watch(&source.Kind{Type: &corev1.Node{}}, &handler.EnqueueRequestForObject{}) //if err != nil { // return err @@ -104,11 +106,6 @@ func add(mgr manager.Manager, cfg *appconfig.CompletedConfig, r reconcile.Reconc //return err } -func (r *ReconcilePodBinding) InjectClient(c client.Client) error { - r.Client = c - return nil -} - // Reconcile reads that state of Node in cluster and makes changes if node autonomy state has been changed func (r *ReconcilePodBinding) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { var err error diff --git a/pkg/yurtmanager/controller/yurtcoordinator/podbinding/podbinding_controller_test.go b/pkg/yurtmanager/controller/yurtcoordinator/podbinding/podbinding_controller_test.go index 7a18b7722c3..42be82c65aa 100644 --- a/pkg/yurtmanager/controller/yurtcoordinator/podbinding/podbinding_controller_test.go +++ b/pkg/yurtmanager/controller/yurtcoordinator/podbinding/podbinding_controller_test.go @@ -28,10 +28,8 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" nodeutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/node" ) @@ -152,7 +150,7 @@ func TestReconcile(t *testing.T) { if err := clientgoscheme.AddToScheme(scheme); err != nil { t.Fatal("Fail to add kubernetes clint-go custom resource") } - c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(pods...).WithObjects(nodes...).Build() + c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(pods...).WithIndex(&corev1.Pod{}, "spec.nodeName", podIndexer).WithObjects(nodes...).Build() for i := range TestNodesName { var req = reconcile.Request{NamespacedName: types.NamespacedName{Name: TestNodesName[i]}} @@ -246,9 +244,20 @@ func TestConfigureTolerationForPod(t *testing.T) { } } +func podIndexer(rawObj client.Object) []string { + pod, ok := rawObj.(*corev1.Pod) + if !ok { + return []string{} + } + if len(pod.Spec.NodeName) == 0 { + return []string{} + } + return []string{pod.Spec.NodeName} +} + func TestGetPodsAssignedToNode(t *testing.T) { pods := preparePods() - c := fakeclient.NewClientBuilder().WithObjects(pods...).Build() + c := fakeclient.NewClientBuilder().WithObjects(pods...).WithIndex(&corev1.Pod{}, "spec.nodeName", podIndexer).Build() tests := []struct { name string nodeName string @@ -259,10 +268,8 @@ func TestGetPodsAssignedToNode(t *testing.T) { name: "test1", nodeName: "node1", want: []corev1.Pod{ - *pods[0].(*corev1.Pod), *pods[1].(*corev1.Pod), *pods[2].(*corev1.Pod), - *pods[3].(*corev1.Pod), }, wantErr: false, }, @@ -280,7 +287,7 @@ func TestGetPodsAssignedToNode(t *testing.T) { return } if !reflect.DeepEqual(got, tt.want) { - t.Errorf("getPodsAssignedToNode() got = %v, want %v", got, tt.want) + t.Errorf("getPodsAssignedToNode() got = %v\n, want %v\n", got, tt.want) } }) } @@ -405,26 +412,3 @@ func TestIsPodBoundenToNode(t *testing.T) { }) } } - -func Test_newReconciler(t *testing.T) { - tests := []struct { - name string - in0 *appconfig.CompletedConfig - mgr manager.Manager - want reconcile.Reconciler - }{ - { - name: "test1", - in0: nil, - mgr: nil, - want: &ReconcilePodBinding{}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := newReconciler(tt.in0, tt.mgr); !reflect.DeepEqual(got, tt.want) { - t.Errorf("newReconciler() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/pkg/yurtmanager/controller/yurtstaticset/upgradeinfo/upgrade_info_test.go b/pkg/yurtmanager/controller/yurtstaticset/upgradeinfo/upgrade_info_test.go index 17cc3fd471c..9ba2519898c 100644 --- a/pkg/yurtmanager/controller/yurtstaticset/upgradeinfo/upgrade_info_test.go +++ b/pkg/yurtmanager/controller/yurtstaticset/upgradeinfo/upgrade_info_test.go @@ -274,7 +274,7 @@ var ( }, HostNetwork: true, PriorityClassName: "system-node-critical", - Priority: utilpointer.Int32Ptr(2000001000), + Priority: utilpointer.Int32(2000001000), }, }, }, @@ -306,7 +306,7 @@ func preparePods() []*corev1.Pod { }, HostNetwork: true, PriorityClassName: "system-node-critical", - Priority: utilpointer.Int32Ptr(2000001000), + Priority: utilpointer.Int32(2000001000), NodeName: "aaa", SchedulerName: "default-scheduler", RestartPolicy: "Always", @@ -332,7 +332,7 @@ func preparePods() []*corev1.Pod { }, }, PriorityClassName: "system-node-critical", - Priority: utilpointer.Int32Ptr(2000001000), + Priority: utilpointer.Int32(2000001000), NodeName: "aaa", SchedulerName: "default-scheduler", RestartPolicy: "Always", diff --git a/pkg/yurtmanager/controller/yurtstaticset/util/util.go b/pkg/yurtmanager/controller/yurtstaticset/util/util.go index 7e5cfb3553c..f7614b07c15 100644 --- a/pkg/yurtmanager/controller/yurtstaticset/util/util.go +++ b/pkg/yurtmanager/controller/yurtstaticset/util/util.go @@ -128,7 +128,7 @@ func SetPodUpgradeCondition(c client.Client, status corev1.ConditionStatus, pod Status: status, } if change := podutil.UpdatePodCondition(&pod.Status, cond); change { - if err := c.Status().Update(context.TODO(), pod, &client.UpdateOptions{}); err != nil { + if err := c.Status().Update(context.TODO(), pod, &client.SubResourceUpdateOptions{}); err != nil { return err } } diff --git a/pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller.go b/pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller.go index 30394ef6af9..c1d7e8cd6cd 100644 --- a/pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller.go +++ b/pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller.go @@ -159,7 +159,7 @@ func add(mgr manager.Manager, cfg *appconfig.CompletedConfig, r reconcile.Reconc } // 1. Watch for changes to YurtStaticSet - if err := c.Watch(&source.Kind{Type: &appsv1alpha1.YurtStaticSet{}}, &handler.EnqueueRequestForObject{}); err != nil { + if err := c.Watch(source.Kind(mgr.GetCache(), &appsv1alpha1.YurtStaticSet{}), &handler.EnqueueRequestForObject{}); err != nil { return err } @@ -196,16 +196,17 @@ func add(mgr manager.Manager, cfg *appconfig.CompletedConfig, r reconcile.Reconc return requests } - if err := c.Watch(&source.Kind{Type: &corev1.Node{}}, + if err := c.Watch(source.Kind(mgr.GetCache(), &corev1.Node{}), handler.EnqueueRequestsFromMapFunc( - func(client.Object) []reconcile.Request { + func(context.Context, client.Object) []reconcile.Request { return reconcileAllYurtStaticSets(mgr.GetClient()) }), nodeReadyPredicate); err != nil { return err } // 3. Watch for changes to upgrade worker pods which are created by yurt-static-set-controller - if err := c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForOwner{IsController: true, OwnerType: &appsv1alpha1.YurtStaticSet{}}); err != nil { + if err := c.Watch(source.Kind(mgr.GetCache(), &corev1.Pod{}), + handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &appsv1alpha1.YurtStaticSet{}, handler.OnlyControllerOwner())); err != nil { return err } @@ -229,8 +230,8 @@ func add(mgr manager.Manager, cfg *appconfig.CompletedConfig, r reconcile.Reconc return reqs } - if err := c.Watch(&source.Kind{Type: &corev1.Pod{}}, handler.EnqueueRequestsFromMapFunc( - func(obj client.Object) []reconcile.Request { + if err := c.Watch(source.Kind(mgr.GetCache(), &corev1.Pod{}), handler.EnqueueRequestsFromMapFunc( + func(ctx context.Context, obj client.Object) []reconcile.Request { return reconcileYurtStaticSetForStaticPod(obj) })); err != nil { return err diff --git a/pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller_test.go b/pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller_test.go index 4508aa78b43..acad082e858 100644 --- a/pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller_test.go +++ b/pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller_test.go @@ -113,7 +113,7 @@ func TestReconcile(t *testing.T) { for _, s := range strategy { instance.Spec.UpgradeStrategy = s - c := fakeclient.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(instance).WithObjects(staticPods...).WithObjects(nodes...).Build() + c := fakeclient.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(instance).WithStatusSubresource(instance).WithObjects(staticPods...).WithObjects(nodes...).Build() var req = reconcile.Request{NamespacedName: types.NamespacedName{Namespace: metav1.NamespaceDefault, Name: TestStaticPodName}} rsp := ReconcileYurtStaticSet{ @@ -123,7 +123,7 @@ func TestReconcile(t *testing.T) { _, err := rsp.Reconcile(context.TODO(), req) if err != nil { - t.Fatalf("failed to control static-pod controller") + t.Fatalf("failed to control static-pod controller, %v", err) } } } diff --git a/pkg/yurtmanager/webhook/builder/defaulter_custom.go b/pkg/yurtmanager/webhook/builder/defaulter_custom.go deleted file mode 100644 index 349c921dada..00000000000 --- a/pkg/yurtmanager/webhook/builder/defaulter_custom.go +++ /dev/null @@ -1,98 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package builder - -import ( - "context" - "encoding/json" - "errors" - "net/http" - - admissionv1 "k8s.io/api/admission/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - -// CustomDefaulter defines functions for setting defaults on resources. -type CustomDefaulter interface { - Default(ctx context.Context, obj runtime.Object, req admission.Request) error -} - -// WithCustomDefaulter creates a new Webhook for a CustomDefaulter interface. -func WithCustomDefaulter(obj runtime.Object, defaulter CustomDefaulter) *admission.Webhook { - return &admission.Webhook{ - Handler: &defaulterForType{object: obj, defaulter: defaulter}, - } -} - -type defaulterForType struct { - defaulter CustomDefaulter - object runtime.Object - decoder *admission.Decoder -} - -var _ admission.DecoderInjector = &defaulterForType{} - -func (h *defaulterForType) InjectDecoder(d *admission.Decoder) error { - h.decoder = d - return nil -} - -// Handle handles admission requests. -func (h *defaulterForType) Handle(ctx context.Context, req admission.Request) admission.Response { - if h.defaulter == nil { - panic("defaulter should never be nil") - } - if h.object == nil { - panic("object should never be nil") - } - - // Get the object in the request - obj := h.object.DeepCopyObject() - if err := h.decoder.Decode(req, obj); err != nil { - return admission.Errored(http.StatusBadRequest, err) - } - - // Default the object - if err := h.defaulter.Default(ctx, obj, req); err != nil { - var apiStatus apierrors.APIStatus - if errors.As(err, &apiStatus) { - return validationResponseFromStatus(false, apiStatus.Status()) - } - return admission.Denied(err.Error()) - } - - // Create the patch - marshalled, err := json.Marshal(obj) - if err != nil { - return admission.Errored(http.StatusInternalServerError, err) - } - return admission.PatchResponseFromRaw(req.Object.Raw, marshalled) -} - -// validationResponseFromStatus returns a response for admitting a request with provided Status object. -func validationResponseFromStatus(allowed bool, status metav1.Status) admission.Response { - resp := admission.Response{ - AdmissionResponse: admissionv1.AdmissionResponse{ - Allowed: allowed, - Result: &status, - }, - } - return resp -} diff --git a/pkg/yurtmanager/webhook/builder/validator_custom.go b/pkg/yurtmanager/webhook/builder/validator_custom.go deleted file mode 100644 index 1ca9a6594d3..00000000000 --- a/pkg/yurtmanager/webhook/builder/validator_custom.go +++ /dev/null @@ -1,112 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package builder - -import ( - "context" - "errors" - "fmt" - "net/http" - - v1 "k8s.io/api/admission/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - -// CustomValidator defines functions for validating an operation. -type CustomValidator interface { - ValidateCreate(ctx context.Context, obj runtime.Object, req admission.Request) error - ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object, req admission.Request) error - ValidateDelete(ctx context.Context, obj runtime.Object, req admission.Request) error -} - -// WithCustomValidator creates a new Webhook for validating the provided type. -func WithCustomValidator(obj runtime.Object, validator CustomValidator) *admission.Webhook { - return &admission.Webhook{ - Handler: &validatorForType{object: obj, validator: validator}, - } -} - -type validatorForType struct { - validator CustomValidator - object runtime.Object - decoder *admission.Decoder -} - -var _ admission.DecoderInjector = &validatorForType{} - -// InjectDecoder injects the decoder into a validatingHandler. -func (h *validatorForType) InjectDecoder(d *admission.Decoder) error { - h.decoder = d - return nil -} - -// Handle handles admission requests. -func (h *validatorForType) Handle(ctx context.Context, req admission.Request) admission.Response { - if h.validator == nil { - panic("validator should never be nil") - } - if h.object == nil { - panic("object should never be nil") - } - - // Get the object in the request - obj := h.object.DeepCopyObject() - - var err error - switch req.Operation { - case v1.Create: - if err := h.decoder.Decode(req, obj); err != nil { - return admission.Errored(http.StatusBadRequest, err) - } - - err = h.validator.ValidateCreate(ctx, obj, req) - case v1.Update: - oldObj := obj.DeepCopyObject() - if err := h.decoder.DecodeRaw(req.Object, obj); err != nil { - return admission.Errored(http.StatusBadRequest, err) - } - if err := h.decoder.DecodeRaw(req.OldObject, oldObj); err != nil { - return admission.Errored(http.StatusBadRequest, err) - } - - err = h.validator.ValidateUpdate(ctx, oldObj, obj, req) - case v1.Delete: - // In reference to PR: https://github.com/kubernetes/kubernetes/pull/76346 - // OldObject contains the object being deleted - if err := h.decoder.DecodeRaw(req.OldObject, obj); err != nil { - return admission.Errored(http.StatusBadRequest, err) - } - - err = h.validator.ValidateDelete(ctx, obj, req) - default: - return admission.Errored(http.StatusBadRequest, fmt.Errorf("unknown operation request %q", req.Operation)) - } - - // Check the error message first. - if err != nil { - var apiStatus apierrors.APIStatus - if errors.As(err, &apiStatus) { - return validationResponseFromStatus(false, apiStatus.Status()) - } - return admission.Denied(err.Error()) - } - - // Return allowed if everything succeeded. - return admission.Allowed("") -} diff --git a/pkg/yurtmanager/webhook/builder/webhook.go b/pkg/yurtmanager/webhook/builder/webhook.go deleted file mode 100644 index a70bcbe4833..00000000000 --- a/pkg/yurtmanager/webhook/builder/webhook.go +++ /dev/null @@ -1,199 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package builder - -import ( - "errors" - "net/http" - "net/url" - - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/rest" - "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - "sigs.k8s.io/controller-runtime/pkg/webhook/conversion" - - "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" -) - -// WebhookBuilder builds a Webhook. -type WebhookBuilder struct { - apiType runtime.Object - withDefaulter CustomDefaulter - withValidator CustomValidator - gvk schema.GroupVersionKind - mgr manager.Manager - config *rest.Config -} - -// WebhookManagedBy allows inform its manager.Manager. -func WebhookManagedBy(m manager.Manager) *WebhookBuilder { - return &WebhookBuilder{mgr: m} -} - -// For takes a runtime.Object which should be a CR. -// If the given object implements the admission.Defaulter interface, a MutatingWebhook will be wired for this type. -// If the given object implements the admission.Validator interface, a ValidatingWebhook will be wired for this type. -func (blder *WebhookBuilder) For(apiType runtime.Object) *WebhookBuilder { - blder.apiType = apiType - return blder -} - -// WithDefaulter takes a WithDefaulter interface, a MutatingWebhook will be wired for this type. -func (blder *WebhookBuilder) WithDefaulter(defaulter CustomDefaulter) *WebhookBuilder { - blder.withDefaulter = defaulter - return blder -} - -// WithValidator takes a WithValidator interface, a ValidatingWebhook will be wired for this type. -func (blder *WebhookBuilder) WithValidator(validator CustomValidator) *WebhookBuilder { - blder.withValidator = validator - return blder -} - -// Complete builds the webhook. -func (blder *WebhookBuilder) Complete() error { - // Set the Config - blder.loadRestConfig() - - // Set the Webhook if needed - return blder.registerWebhooks() -} - -func (blder *WebhookBuilder) loadRestConfig() { - if blder.config == nil { - blder.config = blder.mgr.GetConfig() - } -} - -func (blder *WebhookBuilder) registerWebhooks() error { - typ, err := blder.getType() - if err != nil { - return err - } - - // Create webhook(s) for each type - blder.gvk, err = apiutil.GVKForObject(typ, blder.mgr.GetScheme()) - if err != nil { - return err - } - - blder.registerDefaultingWebhook() - blder.registerValidatingWebhook() - - err = blder.registerConversionWebhook() - if err != nil { - return err - } - return nil -} - -func (blder *WebhookBuilder) getType() (runtime.Object, error) { - if blder.apiType != nil { - return blder.apiType, nil - } - return nil, errors.New("for() must be called with a valid object") -} - -// registerDefaultingWebhook registers a defaulting webhook if th. -func (blder *WebhookBuilder) registerDefaultingWebhook() { - mwh := blder.getDefaultingWebhook() - if mwh != nil { - path := util.GenerateMutatePath(blder.gvk) - - // Checking if the path is already registered. - // If so, just skip it. - if !blder.isAlreadyHandled(path) { - klog.Info("Registering a mutating webhook", - "GVK", blder.gvk, - "path", path) - blder.mgr.GetWebhookServer().Register(path, mwh) - } - } -} - -func (blder *WebhookBuilder) getDefaultingWebhook() *admission.Webhook { - if defaulter := blder.withDefaulter; defaulter != nil { - return WithCustomDefaulter(blder.apiType, defaulter) - } - if defaulter, ok := blder.apiType.(admission.Defaulter); ok { - return admission.DefaultingWebhookFor(defaulter) - } - klog.Info( - "skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called", - "GVK", blder.gvk) - return nil -} - -func (blder *WebhookBuilder) isAlreadyHandled(path string) bool { - if blder.mgr.GetWebhookServer().WebhookMux == nil { - return false - } - h, p := blder.mgr.GetWebhookServer().WebhookMux.Handler(&http.Request{URL: &url.URL{Path: path}}) - if p == path && h != nil { - return true - } - return false -} - -func (blder *WebhookBuilder) registerValidatingWebhook() { - vwh := blder.getValidatingWebhook() - if vwh != nil { - path := util.GenerateValidatePath(blder.gvk) - - // Checking if the path is already registered. - // If so, just skip it. - if !blder.isAlreadyHandled(path) { - klog.Info("Registering a validating webhook", - "GVK", blder.gvk, - "path", path) - blder.mgr.GetWebhookServer().Register(path, vwh) - } - } -} - -func (blder *WebhookBuilder) getValidatingWebhook() *admission.Webhook { - if validator := blder.withValidator; validator != nil { - return WithCustomValidator(blder.apiType, validator) - } - if validator, ok := blder.apiType.(admission.Validator); ok { - return admission.ValidatingWebhookFor(validator) - } - klog.Info( - "skip registering a validating webhook, object does not implement admission.Validator or WithValidator wasn't called", - "GVK", blder.gvk) - return nil -} - -func (blder *WebhookBuilder) registerConversionWebhook() error { - ok, err := conversion.IsConvertible(blder.mgr.GetScheme(), blder.apiType) - if err != nil { - klog.Error(err, "conversion check failed", "GVK", blder.gvk) - return err - } - if ok { - if !blder.isAlreadyHandled("/convert") { - blder.mgr.GetWebhookServer().Register("/convert", &conversion.Webhook{}) - } - klog.Info("Conversion webhook enabled", "GVK", blder.gvk) - } - - return nil -} diff --git a/pkg/yurtmanager/webhook/gateway/v1alpha1/gateway_validation.go b/pkg/yurtmanager/webhook/gateway/v1alpha1/gateway_validation.go index c1c521682a1..1cc7d151180 100644 --- a/pkg/yurtmanager/webhook/gateway/v1alpha1/gateway_validation.go +++ b/pkg/yurtmanager/webhook/gateway/v1alpha1/gateway_validation.go @@ -26,49 +26,50 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/openyurtio/openyurt/pkg/apis/raven/v1alpha1" ) // ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *GatewayHandler) ValidateCreate(ctx context.Context, obj runtime.Object) error { +func (webhook *GatewayHandler) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { gw, ok := obj.(*v1alpha1.Gateway) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a Gateway but got a %T", obj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Gateway but got a %T", obj)) } - return validate(gw) + return nil, validate(gw) } // ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *GatewayHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error { +func (webhook *GatewayHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { newGw, ok := newObj.(*v1alpha1.Gateway) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a Gateway but got a %T", newObj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Gateway but got a %T", newObj)) } oldGw, ok := oldObj.(*v1alpha1.Gateway) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a Gateway} but got a %T", oldObj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Gateway} but got a %T", oldObj)) } if err := validate(oldGw); err != nil { - return err + return nil, err } if err := validate(newGw); err != nil { - return err + return nil, err } - return nil + return nil, nil } // ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *GatewayHandler) ValidateDelete(_ context.Context, obj runtime.Object) error { +func (webhook *GatewayHandler) ValidateDelete(_ context.Context, obj runtime.Object) (admission.Warnings, error) { gw, ok := obj.(*v1alpha1.Gateway) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a Gateway but got a %T", obj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Gateway but got a %T", obj)) } - return validate(gw) + return nil, validate(gw) } func validate(g *v1alpha1.Gateway) error { diff --git a/pkg/yurtmanager/webhook/gateway/v1beta1/gateway_validation.go b/pkg/yurtmanager/webhook/gateway/v1beta1/gateway_validation.go index b75aafa4bdc..184ada3b3cd 100644 --- a/pkg/yurtmanager/webhook/gateway/v1beta1/gateway_validation.go +++ b/pkg/yurtmanager/webhook/gateway/v1beta1/gateway_validation.go @@ -26,47 +26,44 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" ) // ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *GatewayHandler) ValidateCreate(ctx context.Context, obj runtime.Object) error { +func (webhook *GatewayHandler) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { gw, ok := obj.(*v1beta1.Gateway) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a Gateway but got a %T", obj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Gateway but got a %T", obj)) } return validate(gw) } // ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *GatewayHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error { +func (webhook *GatewayHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { newGw, ok := newObj.(*v1beta1.Gateway) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a Gateway but got a %T", newObj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Gateway but got a %T", newObj)) } oldGw, ok := oldObj.(*v1beta1.Gateway) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a Gateway} but got a %T", oldObj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Gateway} but got a %T", oldObj)) } if newGw.GetName() != oldGw.GetName() { - return apierrors.NewBadRequest(fmt.Sprintf("gateway name can not change")) + return nil, apierrors.NewBadRequest(fmt.Sprintf("gateway name can not change")) } - if err := validate(newGw); err != nil { - return err - } - - return nil + return validate(newGw) } // ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *GatewayHandler) ValidateDelete(_ context.Context, obj runtime.Object) error { - return nil +func (webhook *GatewayHandler) ValidateDelete(_ context.Context, obj runtime.Object) (admission.Warnings, error) { + return nil, nil } -func validate(g *v1beta1.Gateway) error { +func validate(g *v1beta1.Gateway) (admission.Warnings, error) { var errList field.ErrorList if g.Spec.ExposeType != "" { @@ -129,14 +126,14 @@ func validate(g *v1beta1.Gateway) error { } if errList != nil { - return apierrors.NewInvalid( + return nil, apierrors.NewInvalid( schema.GroupKind{Group: v1beta1.SchemeGroupVersion.Group, Kind: g.Kind}, g.Name, errList) } klog.Infof("Validate Gateway %s successfully ...", klog.KObj(g)) - return nil + return nil, nil } func validateIP(ip string) error { diff --git a/pkg/yurtmanager/webhook/node/v1/node_default.go b/pkg/yurtmanager/webhook/node/v1/node_default.go index ec967896853..b680e190d25 100644 --- a/pkg/yurtmanager/webhook/node/v1/node_default.go +++ b/pkg/yurtmanager/webhook/node/v1/node_default.go @@ -24,7 +24,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/openyurtio/openyurt/pkg/apis/apps" appsv1beta1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" @@ -32,7 +31,7 @@ import ( ) // Default satisfies the defaulting webhook interface. -func (webhook *NodeHandler) Default(ctx context.Context, obj runtime.Object, req admission.Request) error { +func (webhook *NodeHandler) Default(ctx context.Context, obj runtime.Object) error { node, ok := obj.(*v1.Node) if !ok { return apierrors.NewBadRequest(fmt.Sprintf("expected a Node but got a %T", obj)) diff --git a/pkg/yurtmanager/webhook/node/v1/node_default_test.go b/pkg/yurtmanager/webhook/node/v1/node_default_test.go index 7ff73778cab..8dbf43d3c7a 100644 --- a/pkg/yurtmanager/webhook/node/v1/node_default_test.go +++ b/pkg/yurtmanager/webhook/node/v1/node_default_test.go @@ -29,7 +29,6 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/openyurtio/openyurt/pkg/apis" appsv1beta1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" @@ -101,7 +100,7 @@ func TestDefault(t *testing.T) { } w := &NodeHandler{Client: c} - err := w.Default(context.TODO(), tc.node, admission.Request{}) + err := w.Default(context.TODO(), tc.node) if err != nil { if tc.errCode != 0 { statusErr := err.(*errors.StatusError) diff --git a/pkg/yurtmanager/webhook/node/v1/node_handler.go b/pkg/yurtmanager/webhook/node/v1/node_handler.go index 707b7de1278..101afe40f8c 100644 --- a/pkg/yurtmanager/webhook/node/v1/node_handler.go +++ b/pkg/yurtmanager/webhook/node/v1/node_handler.go @@ -21,8 +21,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/webhook" - "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/builder" "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" ) @@ -41,7 +41,7 @@ func (webhook *NodeHandler) SetupWebhookWithManager(mgr ctrl.Manager) (string, s } return util.GenerateMutatePath(gvk), util.GenerateValidatePath(gvk), - builder.WebhookManagedBy(mgr). + ctrl.NewWebhookManagedBy(mgr). For(&v1.Node{}). WithDefaulter(webhook). WithValidator(webhook). @@ -56,5 +56,5 @@ type NodeHandler struct { Client client.Client } -var _ builder.CustomDefaulter = &NodeHandler{} -var _ builder.CustomValidator = &NodeHandler{} +var _ webhook.CustomDefaulter = &NodeHandler{} +var _ webhook.CustomValidator = &NodeHandler{} diff --git a/pkg/yurtmanager/webhook/node/v1/node_validation.go b/pkg/yurtmanager/webhook/node/v1/node_validation.go index cd7262914a8..d69b59af308 100644 --- a/pkg/yurtmanager/webhook/node/v1/node_validation.go +++ b/pkg/yurtmanager/webhook/node/v1/node_validation.go @@ -31,34 +31,34 @@ import ( ) // ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *NodeHandler) ValidateCreate(_ context.Context, obj runtime.Object, req admission.Request) error { - return nil +func (webhook *NodeHandler) ValidateCreate(_ context.Context, obj runtime.Object) (admission.Warnings, error) { + return nil, nil } // ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *NodeHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object, req admission.Request) error { +func (webhook *NodeHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { newNode, ok := newObj.(*v1.Node) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a Node but got a %T", newObj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Node but got a %T", newObj)) } oldNode, ok := oldObj.(*v1.Node) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a Node} but got a %T", oldObj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Node} but got a %T", oldObj)) } - if allErrs := validateNodeUpdate(newNode, oldNode, req); len(allErrs) > 0 { - return apierrors.NewInvalid(v1.SchemeGroupVersion.WithKind("Node").GroupKind(), newNode.Name, allErrs) + if allErrs := validateNodeUpdate(newNode, oldNode); len(allErrs) > 0 { + return nil, apierrors.NewInvalid(v1.SchemeGroupVersion.WithKind("Node").GroupKind(), newNode.Name, allErrs) } - return nil + return nil, nil } // ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *NodeHandler) ValidateDelete(_ context.Context, obj runtime.Object, req admission.Request) error { - return nil +func (webhook *NodeHandler) ValidateDelete(_ context.Context, obj runtime.Object) (admission.Warnings, error) { + return nil, nil } -func validateNodeUpdate(newNode, oldNode *v1.Node, req admission.Request) field.ErrorList { +func validateNodeUpdate(newNode, oldNode *v1.Node) field.ErrorList { oldNp := oldNode.Labels[projectinfo.GetNodePoolLabel()] newNp := newNode.Labels[projectinfo.GetNodePoolLabel()] oldNpHostNetwork := oldNode.Labels[apps.NodePoolHostNetworkLabel] diff --git a/pkg/yurtmanager/webhook/node/v1/node_validation_test.go b/pkg/yurtmanager/webhook/node/v1/node_validation_test.go index d8f8253a25d..35fec500623 100644 --- a/pkg/yurtmanager/webhook/node/v1/node_validation_test.go +++ b/pkg/yurtmanager/webhook/node/v1/node_validation_test.go @@ -25,7 +25,6 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/openyurtio/openyurt/pkg/apis/apps" "github.com/openyurtio/openyurt/pkg/projectinfo" @@ -119,7 +118,7 @@ func TestValidateUpdate(t *testing.T) { for k, tc := range testcases { t.Run(k, func(t *testing.T) { h := &NodeHandler{} - err := h.ValidateUpdate(context.TODO(), tc.oldNode, tc.newNode, admission.Request{}) + _, err := h.ValidateUpdate(context.TODO(), tc.oldNode, tc.newNode) if tc.errCode == 0 && err != nil { t.Errorf("Expected error code %d, got %v", tc.errCode, err) } else if tc.errCode != 0 { diff --git a/pkg/yurtmanager/webhook/nodepool/v1beta1/nodepool_validation.go b/pkg/yurtmanager/webhook/nodepool/v1beta1/nodepool_validation.go index a2a18336cab..f7a6a440ecb 100644 --- a/pkg/yurtmanager/webhook/nodepool/v1beta1/nodepool_validation.go +++ b/pkg/yurtmanager/webhook/nodepool/v1beta1/nodepool_validation.go @@ -27,54 +27,55 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" appsv1beta1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" "github.com/openyurtio/openyurt/pkg/projectinfo" ) // ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *NodePoolHandler) ValidateCreate(ctx context.Context, obj runtime.Object) error { +func (webhook *NodePoolHandler) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { np, ok := obj.(*appsv1beta1.NodePool) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a NodePool but got a %T", obj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a NodePool but got a %T", obj)) } if allErrs := validateNodePoolSpec(&np.Spec); len(allErrs) > 0 { - return apierrors.NewInvalid(appsv1beta1.GroupVersion.WithKind("NodePool").GroupKind(), np.Name, allErrs) + return nil, apierrors.NewInvalid(appsv1beta1.GroupVersion.WithKind("NodePool").GroupKind(), np.Name, allErrs) } - return nil + return nil, nil } // ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *NodePoolHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error { +func (webhook *NodePoolHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { newNp, ok := newObj.(*appsv1beta1.NodePool) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a NodePool but got a %T", newObj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a NodePool but got a %T", newObj)) } oldNp, ok := oldObj.(*appsv1beta1.NodePool) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a NodePool but got a %T", oldObj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a NodePool but got a %T", oldObj)) } if allErrs := validateNodePoolSpecUpdate(&newNp.Spec, &oldNp.Spec); len(allErrs) > 0 { - return apierrors.NewForbidden(appsv1beta1.GroupVersion.WithResource("nodepools").GroupResource(), newNp.Name, allErrs[0]) + return nil, apierrors.NewForbidden(appsv1beta1.GroupVersion.WithResource("nodepools").GroupResource(), newNp.Name, allErrs[0]) } - return nil + return nil, nil } // ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *NodePoolHandler) ValidateDelete(_ context.Context, obj runtime.Object) error { +func (webhook *NodePoolHandler) ValidateDelete(_ context.Context, obj runtime.Object) (admission.Warnings, error) { np, ok := obj.(*appsv1beta1.NodePool) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a NodePool but got a %T", obj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a NodePool but got a %T", obj)) } if allErrs := validateNodePoolDeletion(webhook.Client, np); len(allErrs) > 0 { - return apierrors.NewForbidden(appsv1beta1.GroupVersion.WithResource("nodepools").GroupResource(), np.Name, allErrs[0]) + return nil, apierrors.NewForbidden(appsv1beta1.GroupVersion.WithResource("nodepools").GroupResource(), np.Name, allErrs[0]) } - return nil + return nil, nil } // annotationValidator validates the NodePool.Spec.Annotations diff --git a/pkg/yurtmanager/webhook/nodepool/v1beta1/nodepool_validation_test.go b/pkg/yurtmanager/webhook/nodepool/v1beta1/nodepool_validation_test.go index b2896dac51e..4e831a29fef 100644 --- a/pkg/yurtmanager/webhook/nodepool/v1beta1/nodepool_validation_test.go +++ b/pkg/yurtmanager/webhook/nodepool/v1beta1/nodepool_validation_test.go @@ -75,7 +75,7 @@ func TestValidateCreate(t *testing.T) { handler := &NodePoolHandler{} for k, tc := range testcases { t.Run(k, func(t *testing.T) { - err := handler.ValidateCreate(context.TODO(), tc.pool) + _, err := handler.ValidateCreate(context.TODO(), tc.pool) if tc.errcode == 0 && err != nil { t.Errorf("Expected error code %d, got %v", tc.errcode, err) } else if tc.errcode != 0 { @@ -166,7 +166,7 @@ func TestValidateUpdate(t *testing.T) { handler := &NodePoolHandler{} for k, tc := range testcases { t.Run(k, func(t *testing.T) { - err := handler.ValidateUpdate(context.TODO(), tc.oldPool, tc.newPool) + _, err := handler.ValidateUpdate(context.TODO(), tc.oldPool, tc.newPool) if tc.errcode == 0 && err != nil { t.Errorf("Expected error code %d, got %v", tc.errcode, err) } else if tc.errcode != 0 { @@ -271,7 +271,7 @@ func TestValidateDelete(t *testing.T) { } for k, tc := range testcases { t.Run(k, func(t *testing.T) { - err := handler.ValidateDelete(context.TODO(), tc.pool) + _, err := handler.ValidateDelete(context.TODO(), tc.pool) if tc.errcode == 0 && err != nil { t.Errorf("Expected error code %d, got %v", tc.errcode, err) } else if tc.errcode != 0 { diff --git a/pkg/yurtmanager/webhook/platformadmin/v1alpha1/platformadmin_validation.go b/pkg/yurtmanager/webhook/platformadmin/v1alpha1/platformadmin_validation.go index fa4236bf794..7bc0422978b 100644 --- a/pkg/yurtmanager/webhook/platformadmin/v1alpha1/platformadmin_validation.go +++ b/pkg/yurtmanager/webhook/platformadmin/v1alpha1/platformadmin_validation.go @@ -24,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" @@ -31,43 +32,43 @@ import ( ) // ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *PlatformAdminHandler) ValidateCreate(ctx context.Context, obj runtime.Object) error { +func (webhook *PlatformAdminHandler) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { platformAdmin, ok := obj.(*v1alpha1.PlatformAdmin) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a PlatformAdmin but got a %T", obj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a PlatformAdmin but got a %T", obj)) } //validate if allErrs := webhook.validate(ctx, platformAdmin); len(allErrs) > 0 { - return apierrors.NewInvalid(v1alpha1.GroupVersion.WithKind("PlatformAdmin").GroupKind(), platformAdmin.Name, allErrs) + return nil, apierrors.NewInvalid(v1alpha1.GroupVersion.WithKind("PlatformAdmin").GroupKind(), platformAdmin.Name, allErrs) } - return nil + return nil, nil } // ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *PlatformAdminHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error { +func (webhook *PlatformAdminHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { newPlatformAdmin, ok := newObj.(*v1alpha1.PlatformAdmin) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a PlatformAdmin but got a %T", newObj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a PlatformAdmin but got a %T", newObj)) } oldPlatformAdmin, ok := oldObj.(*v1alpha1.PlatformAdmin) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a PlatformAdmin but got a %T", oldObj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a PlatformAdmin but got a %T", oldObj)) } // validate newErrorList := webhook.validate(ctx, newPlatformAdmin) oldErrorList := webhook.validate(ctx, oldPlatformAdmin) if allErrs := append(newErrorList, oldErrorList...); len(allErrs) > 0 { - return apierrors.NewInvalid(v1alpha1.GroupVersion.WithKind("PlatformAdmin").GroupKind(), newPlatformAdmin.Name, allErrs) + return nil, apierrors.NewInvalid(v1alpha1.GroupVersion.WithKind("PlatformAdmin").GroupKind(), newPlatformAdmin.Name, allErrs) } - return nil + return nil, nil } // ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *PlatformAdminHandler) ValidateDelete(_ context.Context, obj runtime.Object) error { - return nil +func (webhook *PlatformAdminHandler) ValidateDelete(_ context.Context, obj runtime.Object) (admission.Warnings, error) { + return nil, nil } func (webhook *PlatformAdminHandler) validate(ctx context.Context, platformAdmin *v1alpha1.PlatformAdmin) field.ErrorList { diff --git a/pkg/yurtmanager/webhook/platformadmin/v1alpha2/platformadmin_validation.go b/pkg/yurtmanager/webhook/platformadmin/v1alpha2/platformadmin_validation.go index 3f5d422cd46..e5ff29546c2 100644 --- a/pkg/yurtmanager/webhook/platformadmin/v1alpha2/platformadmin_validation.go +++ b/pkg/yurtmanager/webhook/platformadmin/v1alpha2/platformadmin_validation.go @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha2" @@ -33,43 +34,43 @@ import ( ) // ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *PlatformAdminHandler) ValidateCreate(ctx context.Context, obj runtime.Object) error { +func (webhook *PlatformAdminHandler) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { platformAdmin, ok := obj.(*v1alpha2.PlatformAdmin) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a PlatformAdmin but got a %T", obj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a PlatformAdmin but got a %T", obj)) } //validate if allErrs := webhook.validate(ctx, platformAdmin); len(allErrs) > 0 { - return apierrors.NewInvalid(v1alpha2.GroupVersion.WithKind("PlatformAdmin").GroupKind(), platformAdmin.Name, allErrs) + return nil, apierrors.NewInvalid(v1alpha2.GroupVersion.WithKind("PlatformAdmin").GroupKind(), platformAdmin.Name, allErrs) } - return nil + return nil, nil } // ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *PlatformAdminHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error { +func (webhook *PlatformAdminHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { newPlatformAdmin, ok := newObj.(*v1alpha2.PlatformAdmin) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a PlatformAdmin but got a %T", newObj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a PlatformAdmin but got a %T", newObj)) } oldPlatformAdmin, ok := oldObj.(*v1alpha2.PlatformAdmin) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a PlatformAdmin but got a %T", oldObj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a PlatformAdmin but got a %T", oldObj)) } // validate newErrorList := webhook.validate(ctx, newPlatformAdmin) oldErrorList := webhook.validate(ctx, oldPlatformAdmin) if allErrs := append(newErrorList, oldErrorList...); len(allErrs) > 0 { - return apierrors.NewInvalid(v1alpha2.GroupVersion.WithKind("PlatformAdmin").GroupKind(), newPlatformAdmin.Name, allErrs) + return nil, apierrors.NewInvalid(v1alpha2.GroupVersion.WithKind("PlatformAdmin").GroupKind(), newPlatformAdmin.Name, allErrs) } - return nil + return nil, nil } // ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *PlatformAdminHandler) ValidateDelete(_ context.Context, obj runtime.Object) error { - return nil +func (webhook *PlatformAdminHandler) ValidateDelete(_ context.Context, obj runtime.Object) (admission.Warnings, error) { + return nil, nil } func (webhook *PlatformAdminHandler) validate(ctx context.Context, platformAdmin *v1alpha2.PlatformAdmin) field.ErrorList { @@ -101,7 +102,7 @@ func (webhook *PlatformAdminHandler) validatePlatformAdminSpec(platformAdmin *v1 } return field.ErrorList{ - field.Invalid(field.NewPath("spec", "version"), platformAdmin.Spec.Version, "must be one of"+strings.Join(config.ExtractVersionsName(webhook.Manifests).List(), ",")), + field.Invalid(field.NewPath("spec", "version"), platformAdmin.Spec.Version, "must be one of"+strings.Join(config.ExtractVersionsName(webhook.Manifests).UnsortedList(), ",")), } } diff --git a/pkg/yurtmanager/webhook/pod/v1alpha1/pod_default.go b/pkg/yurtmanager/webhook/pod/v1alpha1/pod_default.go index 25bf6283126..677e6ca73a3 100644 --- a/pkg/yurtmanager/webhook/pod/v1alpha1/pod_default.go +++ b/pkg/yurtmanager/webhook/pod/v1alpha1/pod_default.go @@ -23,13 +23,12 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/openyurtio/openyurt/pkg/apis/apps" ) // Default implements builder.CustomDefaulter. -func (webhook *PodHandler) Default(ctx context.Context, obj runtime.Object, req admission.Request) error { +func (webhook *PodHandler) Default(ctx context.Context, obj runtime.Object) error { pod, ok := obj.(*corev1.Pod) if !ok { return apierrors.NewBadRequest(fmt.Sprintf("expected a Pod but got a %T", obj)) diff --git a/pkg/yurtmanager/webhook/pod/v1alpha1/pod_default_test.go b/pkg/yurtmanager/webhook/pod/v1alpha1/pod_default_test.go index 0d67c94facd..1039e9fecba 100644 --- a/pkg/yurtmanager/webhook/pod/v1alpha1/pod_default_test.go +++ b/pkg/yurtmanager/webhook/pod/v1alpha1/pod_default_test.go @@ -24,7 +24,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/openyurtio/openyurt/pkg/apis/apps" ) @@ -312,7 +311,7 @@ func TestDefault(t *testing.T) { for k, tc := range testcases { t.Run(k, func(t *testing.T) { h := PodHandler{} - err := h.Default(context.TODO(), tc.obj, admission.Request{}) + err := h.Default(context.TODO(), tc.obj) if tc.errHappened { if err == nil { t.Errorf("expect error, got nil") diff --git a/pkg/yurtmanager/webhook/pod/v1alpha1/pod_handler.go b/pkg/yurtmanager/webhook/pod/v1alpha1/pod_handler.go index 6c8ae786442..79622d8f2f8 100644 --- a/pkg/yurtmanager/webhook/pod/v1alpha1/pod_handler.go +++ b/pkg/yurtmanager/webhook/pod/v1alpha1/pod_handler.go @@ -21,8 +21,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/webhook" - "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/builder" "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" ) @@ -41,7 +41,7 @@ func (webhook *PodHandler) SetupWebhookWithManager(mgr ctrl.Manager) (string, st } return util.GenerateMutatePath(gvk), util.GenerateValidatePath(gvk), - builder.WebhookManagedBy(mgr). + ctrl.NewWebhookManagedBy(mgr). For(&corev1.Pod{}). WithDefaulter(webhook). Complete() @@ -54,4 +54,4 @@ type PodHandler struct { Client client.Client } -var _ builder.CustomDefaulter = &PodHandler{} +var _ webhook.CustomDefaulter = &PodHandler{} diff --git a/pkg/yurtmanager/webhook/util/writer/atomic/atomic_writer.go b/pkg/yurtmanager/webhook/util/writer/atomic/atomic_writer.go index 8e76176be92..b0fd4e098a2 100644 --- a/pkg/yurtmanager/webhook/util/writer/atomic/atomic_writer.go +++ b/pkg/yurtmanager/webhook/util/writer/atomic/atomic_writer.go @@ -139,7 +139,7 @@ func (w *Writer) Write(payload map[string]FileProjection) error { } oldTsPath := path.Join(w.targetDir, oldTsDir) - var pathsToRemove sets.String + var pathsToRemove sets.Set[string] // if there was no old version, there's nothing to remove if len(oldTsDir) != 0 { // (3) @@ -310,8 +310,8 @@ func shouldWriteFile(path string, content []byte) (bool, error) { // pathsToRemove walks the current version of the data directory and // determines which paths should be removed (if any) after the payload is // written to the target directory. -func (w *Writer) pathsToRemove(payload map[string]FileProjection, oldTsDir string) (sets.String, error) { - paths := sets.NewString() +func (w *Writer) pathsToRemove(payload map[string]FileProjection, oldTsDir string) (sets.Set[string], error) { + paths := sets.Set[string]{} visitor := func(path string, info os.FileInfo, err error) error { relativePath := strings.TrimPrefix(path, oldTsDir) relativePath = strings.TrimPrefix(relativePath, string(os.PathSeparator)) @@ -330,7 +330,7 @@ func (w *Writer) pathsToRemove(payload map[string]FileProjection, oldTsDir strin return nil, err } - newPaths := sets.NewString() + newPaths := sets.Set[string]{} for file := range payload { // add all subpaths for the payload to the set of new paths // to avoid attempting to remove non-empty dirs @@ -436,7 +436,7 @@ func (w *Writer) createUserVisibleFiles(payload map[string]FileProjection) error // removeUserVisiblePaths removes the set of paths from the user-visible // portion of the writer's target directory. -func (w *Writer) removeUserVisiblePaths(paths sets.String) error { +func (w *Writer) removeUserVisiblePaths(paths sets.Set[string]) error { ps := string(os.PathSeparator) var lasterr error for p := range paths { diff --git a/pkg/yurtmanager/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_validation.go b/pkg/yurtmanager/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_validation.go index fca4cd2ed70..24995b9bf31 100644 --- a/pkg/yurtmanager/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_validation.go +++ b/pkg/yurtmanager/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_validation.go @@ -35,6 +35,7 @@ import ( corev1 "k8s.io/kubernetes/pkg/apis/core/v1" apivalidation "k8s.io/kubernetes/pkg/apis/core/validation" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" ) @@ -44,41 +45,41 @@ const ( ) // ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *YurtAppDaemonHandler) ValidateCreate(ctx context.Context, obj runtime.Object) error { +func (webhook *YurtAppDaemonHandler) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { daemon, ok := obj.(*v1alpha1.YurtAppDaemon) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppDaemon but got a %T", obj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppDaemon but got a %T", obj)) } if allErrs := validateYurtAppDaemon(webhook.Client, daemon); len(allErrs) > 0 { - return apierrors.NewInvalid(v1alpha1.GroupVersion.WithKind(YurtAppDaemonKind).GroupKind(), daemon.Name, allErrs) + return nil, apierrors.NewInvalid(v1alpha1.GroupVersion.WithKind(YurtAppDaemonKind).GroupKind(), daemon.Name, allErrs) } - return nil + return nil, nil } // ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *YurtAppDaemonHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error { +func (webhook *YurtAppDaemonHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { newDaemon, ok := newObj.(*v1alpha1.YurtAppDaemon) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppDaemon but got a %T", newObj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppDaemon but got a %T", newObj)) } oldDaemon, ok := oldObj.(*v1alpha1.YurtAppDaemon) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppDaemon but got a %T", oldObj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppDaemon but got a %T", oldObj)) } validationErrorList := validateYurtAppDaemon(webhook.Client, newDaemon) updateErrorList := ValidateYurtAppDaemonUpdate(newDaemon, oldDaemon) if allErrs := append(validationErrorList, updateErrorList...); len(allErrs) > 0 { - return apierrors.NewInvalid(v1alpha1.GroupVersion.WithKind(YurtAppDaemonKind).GroupKind(), newDaemon.Name, allErrs) + return nil, apierrors.NewInvalid(v1alpha1.GroupVersion.WithKind(YurtAppDaemonKind).GroupKind(), newDaemon.Name, allErrs) } - return nil + return nil, nil } // ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *YurtAppDaemonHandler) ValidateDelete(_ context.Context, obj runtime.Object) error { - return nil +func (webhook *YurtAppDaemonHandler) ValidateDelete(_ context.Context, obj runtime.Object) (admission.Warnings, error) { + return nil, nil } // validateYurtAppDaemon validates a YurtAppDaemon. @@ -95,7 +96,7 @@ func validateYurtAppDaemonSpec(c client.Client, spec *v1alpha1.YurtAppDaemonSpec if spec.Selector == nil { allErrs = append(allErrs, field.Required(fldPath.Child("selector"), "")) } else { - allErrs = append(allErrs, unversionedvalidation.ValidateLabelSelector(spec.Selector, fldPath.Child("selector"))...) + allErrs = append(allErrs, unversionedvalidation.ValidateLabelSelector(spec.Selector, unversionedvalidation.LabelSelectorValidationOptions{AllowInvalidLabelValueInSelector: false}, fldPath.Child("selector"))...) if len(spec.Selector.MatchLabels)+len(spec.Selector.MatchExpressions) == 0 { allErrs = append(allErrs, field.Invalid(fldPath.Child("selector"), spec.Selector, "empty selector is invalid for statefulset")) } diff --git a/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_validation.go b/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_validation.go index b319bf68fcb..d237ee24fce 100644 --- a/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_validation.go +++ b/pkg/yurtmanager/webhook/yurtappoverrider/v1alpha1/yurtappoverrider_validation.go @@ -24,68 +24,69 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" ) // ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *YurtAppOverriderHandler) ValidateCreate(ctx context.Context, obj runtime.Object) error { +func (webhook *YurtAppOverriderHandler) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { overrider, ok := obj.(*v1alpha1.YurtAppOverrider) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppOverrider but got a %T", obj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppOverrider but got a %T", obj)) } // validate if err := webhook.validateOneToOneBinding(ctx, overrider); err != nil { - return err + return nil, err } - return nil + return nil, nil } // ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *YurtAppOverriderHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error { +func (webhook *YurtAppOverriderHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { oldOverrider, ok := oldObj.(*v1alpha1.YurtAppOverrider) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppOverrider but got a %T", newObj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppOverrider but got a %T", newObj)) } newOverrider, ok := newObj.(*v1alpha1.YurtAppOverrider) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppOverrider} but got a %T", oldObj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppOverrider} but got a %T", oldObj)) } if oldOverrider.Namespace != newOverrider.Namespace || newOverrider.Name != oldOverrider.Name { - return fmt.Errorf("unable to change metadata after %s is created", oldOverrider.Name) + return nil, fmt.Errorf("unable to change metadata after %s is created", oldOverrider.Name) } if newOverrider.Subject.Kind != oldOverrider.Subject.Kind || newOverrider.Subject.Name != oldOverrider.Subject.Name { - return fmt.Errorf("unable to modify subject after %s is created", oldOverrider.Name) + return nil, fmt.Errorf("unable to modify subject after %s is created", oldOverrider.Name) } // validate if err := webhook.validateOneToOneBinding(ctx, newOverrider); err != nil { - return err + return nil, err } - return nil + return nil, nil } // ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *YurtAppOverriderHandler) ValidateDelete(ctx context.Context, obj runtime.Object) error { +func (webhook *YurtAppOverriderHandler) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { overrider, ok := obj.(*v1alpha1.YurtAppOverrider) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppOverrider but got a %T", obj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppOverrider but got a %T", obj)) } switch overrider.Subject.Kind { case "YurtAppSet": appSet := &v1alpha1.YurtAppSet{} err := webhook.Client.Get(ctx, client.ObjectKey{Name: overrider.Subject.Name, Namespace: overrider.Namespace}, appSet) if err == nil { - return fmt.Errorf("namespace: %s, unable to delete YurtAppOverrider when subject resource exists: %s", overrider.Namespace, appSet.Name) + return nil, fmt.Errorf("namespace: %s, unable to delete YurtAppOverrider when subject resource exists: %s", overrider.Namespace, appSet.Name) } case "YurtAppDaemon": appDaemon := &v1alpha1.YurtAppDaemon{} err := webhook.Client.Get(ctx, client.ObjectKey{Name: overrider.Subject.Name, Namespace: overrider.Namespace}, appDaemon) if err == nil { - return fmt.Errorf("namespace: %s, unable to delete YurtAppOverrider when subject resource exists: %s", overrider.Namespace, appDaemon.Name) + return nil, fmt.Errorf("namespace: %s, unable to delete YurtAppOverrider when subject resource exists: %s", overrider.Namespace, appDaemon.Name) } } - return nil + return nil, nil } // YurtAppOverrider and YurtAppSet are one-to-one relationship diff --git a/pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_default.go b/pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_default.go index 2bc67c8c4af..64c3af3b2d7 100644 --- a/pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_default.go +++ b/pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_default.go @@ -36,7 +36,7 @@ func (webhook *YurtAppSetHandler) Default(ctx context.Context, obj runtime.Objec } if set.Spec.RevisionHistoryLimit == nil { - set.Spec.RevisionHistoryLimit = utilpointer.Int32Ptr(10) + set.Spec.RevisionHistoryLimit = utilpointer.Int32(10) klog.V(4).Info("defaulting YurtAppSet.Spec.RevisionHistoryLimit to 10") } diff --git a/pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_validation.go b/pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_validation.go index 3225fa134b7..9755ef4168c 100644 --- a/pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_validation.go +++ b/pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_validation.go @@ -29,6 +29,7 @@ import ( v1 "k8s.io/kubernetes/pkg/apis/apps/v1" appsvalidation "k8s.io/kubernetes/pkg/apis/apps/validation" "k8s.io/kubernetes/pkg/apis/core/validation" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/workloadmanager" @@ -37,73 +38,73 @@ import ( const YurtAppSetKind = "YurtAppSet" // ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *YurtAppSetHandler) ValidateCreate(ctx context.Context, obj runtime.Object) error { +func (webhook *YurtAppSetHandler) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { set, ok := obj.(*v1beta1.YurtAppSet) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppSet but got a %T", obj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppSet but got a %T", obj)) } template := set.Spec.Workload.WorkloadTemplate if template.DeploymentTemplate == nil && template.StatefulSetTemplate == nil { - return apierrors.NewInvalid(v1beta1.GroupVersion.WithKind(YurtAppSetKind).GroupKind(), set.Name, + return nil, apierrors.NewInvalid(v1beta1.GroupVersion.WithKind(YurtAppSetKind).GroupKind(), set.Name, field.ErrorList{field.Invalid(field.NewPath("spec").Child("workload").Child("WorkloadTemplate"), template, "no workload template is configured")}) } else if template.DeploymentTemplate != nil && template.StatefulSetTemplate != nil { - return apierrors.NewInvalid(v1beta1.GroupVersion.WithKind(YurtAppSetKind).GroupKind(), set.Name, + return nil, apierrors.NewInvalid(v1beta1.GroupVersion.WithKind(YurtAppSetKind).GroupKind(), set.Name, field.ErrorList{field.Invalid(field.NewPath("spec").Child("workload").Child("WorkloadTemplate"), template, "only one workload template should be configured")}) } if template.DeploymentTemplate != nil { if err := webhook.validateDeployment(set); err != nil { - return err + return nil, err } } else if template.StatefulSetTemplate != nil { if err := webhook.validateStatefulSet(set); err != nil { - return err + return nil, err } } klog.Infof("Validate YurtAppSet %s successfully ...", klog.KObj(set)) - return nil + return nil, nil } // ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *YurtAppSetHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error { +func (webhook *YurtAppSetHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { newSet, ok := newObj.(*v1beta1.YurtAppSet) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppSet but got a %T", newObj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppSet but got a %T", newObj)) } oldSet, ok := oldObj.(*v1beta1.YurtAppSet) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppSet but got a %T", oldObj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a YurtAppSet but got a %T", oldObj)) } newTemplate := newSet.Spec.Workload.WorkloadTemplate if newTemplate.DeploymentTemplate == nil && newTemplate.StatefulSetTemplate == nil { - return apierrors.NewInvalid(v1beta1.GroupVersion.WithKind(YurtAppSetKind).GroupKind(), newSet.Name, + return nil, apierrors.NewInvalid(v1beta1.GroupVersion.WithKind(YurtAppSetKind).GroupKind(), newSet.Name, field.ErrorList{field.Invalid(field.NewPath("spec").Child("workload").Child("WorkloadTemplate"), newTemplate, "no workload template is configured")}) } else if newTemplate.DeploymentTemplate != nil && newTemplate.StatefulSetTemplate != nil { - return apierrors.NewInvalid(v1beta1.GroupVersion.WithKind(YurtAppSetKind).GroupKind(), newSet.Name, + return nil, apierrors.NewInvalid(v1beta1.GroupVersion.WithKind(YurtAppSetKind).GroupKind(), newSet.Name, field.ErrorList{field.Invalid(field.NewPath("spec").Child("workload").Child("WorkloadTemplate"), newTemplate, "only one workload template should be configured")}) } if newTemplate.DeploymentTemplate != nil { if err := webhook.validateDeployment(newSet); err != nil { - return err + return nil, err } } else if newTemplate.StatefulSetTemplate != nil { if err := webhook.validateStatefulSet(newSet); err != nil { - return err + return nil, err } } oldTemplate := oldSet.Spec.Workload.WorkloadTemplate if (oldTemplate.DeploymentTemplate == nil && newTemplate.DeploymentTemplate != nil) || (oldTemplate.StatefulSetTemplate == nil && newTemplate.StatefulSetTemplate != nil) { - return apierrors.NewInvalid(v1beta1.GroupVersion.WithKind(YurtAppSetKind).GroupKind(), newSet.Name, + return nil, apierrors.NewInvalid(v1beta1.GroupVersion.WithKind(YurtAppSetKind).GroupKind(), newSet.Name, field.ErrorList{field.Invalid(field.NewPath("spec").Child("workload").Child("WorkloadTemplate"), newTemplate, "the kind of workload template should not be changed")}) } - return nil + return nil, nil } // TODO: move functions under k8s.io/kubernetes to pkg/util/kubernetes @@ -116,7 +117,7 @@ func (webhook *YurtAppSetHandler) validateDeployment(yas *v1beta1.YurtAppSet) er if err := v1.Convert_v1_Deployment_To_apps_Deployment(deploy, out, nil); err != nil { return err } - allErrs := appsvalidation.ValidateDeploymentSpec(&out.Spec, field.NewPath("spec"), validation.PodValidationOptions{}) + allErrs := appsvalidation.ValidateDeploymentSpec(&out.Spec, nil, field.NewPath("spec"), validation.PodValidationOptions{}) if len(allErrs) != 0 { return allErrs.ToAggregate() } @@ -135,7 +136,7 @@ func (webhook *YurtAppSetHandler) validateDeployment(yas *v1beta1.YurtAppSet) er if err := v1.Convert_v1_Deployment_To_apps_Deployment(deploy, out, nil); err != nil { return err } - allErrs := appsvalidation.ValidateDeploymentSpec(&out.Spec, field.NewPath("spec"), validation.PodValidationOptions{}) + allErrs := appsvalidation.ValidateDeploymentSpec(&out.Spec, nil, field.NewPath("spec"), validation.PodValidationOptions{}) if len(allErrs) != 0 { return allErrs.ToAggregate() } @@ -179,6 +180,6 @@ func (webhook *YurtAppSetHandler) validateStatefulSet(yas *v1beta1.YurtAppSet) e } // ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *YurtAppSetHandler) ValidateDelete(_ context.Context, obj runtime.Object) error { - return nil +func (webhook *YurtAppSetHandler) ValidateDelete(_ context.Context, obj runtime.Object) (admission.Warnings, error) { + return nil, nil } diff --git a/pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_webhook_test.go b/pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_webhook_test.go index a6527a90120..b1bcfd3825d 100644 --- a/pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_webhook_test.go +++ b/pkg/yurtmanager/webhook/yurtappset/v1beta1/yurtappset_webhook_test.go @@ -109,18 +109,18 @@ func TestYurtAppSetValidator(t *testing.T) { t.Fatal(err) } - if err := webhook.ValidateCreate(context.TODO(), deployAppSet); err != nil { + if _, err := webhook.ValidateCreate(context.TODO(), deployAppSet); err != nil { t.Fatal("yurtappset should create success", err) } dupTopology := deployAppSet.DeepCopy() - if err := webhook.ValidateCreate(context.TODO(), dupTopology); err != nil { + if _, err := webhook.ValidateCreate(context.TODO(), dupTopology); err != nil { t.Fatal("topology dup should not fail") } updateAppSet := deployAppSet.DeepCopy() updateAppSet.Spec.WorkloadTemplate.DeploymentTemplate.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{"app": "demo2"}} - if err := webhook.ValidateUpdate(context.TODO(), deployAppSet, updateAppSet); err == nil { + if _, err := webhook.ValidateUpdate(context.TODO(), deployAppSet, updateAppSet); err == nil { t.Fatal("workload selector should match template selector") } } @@ -137,13 +137,13 @@ func TestYurtAppSetStatefulSetValidator(t *testing.T) { t.Fatal(err) } - if err := webhook.ValidateCreate(context.TODO(), stsAppSet); err != nil { + if _, err := webhook.ValidateCreate(context.TODO(), stsAppSet); err != nil { t.Fatal("yurtappset should create success", err) } updateAppSet := stsAppSet.DeepCopy() updateAppSet.Spec.WorkloadTemplate.StatefulSetTemplate.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{"app": "demo2"}} - if err := webhook.ValidateUpdate(context.TODO(), stsAppSet, updateAppSet); err == nil { + if _, err := webhook.ValidateUpdate(context.TODO(), stsAppSet, updateAppSet); err == nil { t.Fatal("workload selector should match template selector") } } diff --git a/pkg/yurtmanager/webhook/yurtstaticset/v1alpha1/yurtstaticset_validation.go b/pkg/yurtmanager/webhook/yurtstaticset/v1alpha1/yurtstaticset_validation.go index 0a874119497..9f59080d96c 100644 --- a/pkg/yurtmanager/webhook/yurtstaticset/v1alpha1/yurtstaticset_validation.go +++ b/pkg/yurtmanager/webhook/yurtstaticset/v1alpha1/yurtstaticset_validation.go @@ -28,6 +28,7 @@ import ( "k8s.io/kubernetes/pkg/apis/core" k8s_api_v1 "k8s.io/kubernetes/pkg/apis/core/v1" k8s_validation "k8s.io/kubernetes/pkg/apis/core/validation" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" ) @@ -37,40 +38,40 @@ const ( ) // ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *YurtStaticSetHandler) ValidateCreate(ctx context.Context, obj runtime.Object) error { +func (webhook *YurtStaticSetHandler) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { sp, ok := obj.(*v1alpha1.YurtStaticSet) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtStaticSet but got a %T", obj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a YurtStaticSet but got a %T", obj)) } - return validate(sp) + return nil, validate(sp) } // ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *YurtStaticSetHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error { +func (webhook *YurtStaticSetHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { newSP, ok := newObj.(*v1alpha1.YurtStaticSet) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtStaticSet but got a %T", newObj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a YurtStaticSet but got a %T", newObj)) } oldSP, ok := oldObj.(*v1alpha1.YurtStaticSet) if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a YurtStaticSet but got a %T", oldObj)) + return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a YurtStaticSet but got a %T", oldObj)) } if err := validate(newSP); err != nil { - return err + return nil, err } if err := validate(oldSP); err != nil { - return err + return nil, err } - return nil + return nil, nil } // ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *YurtStaticSetHandler) ValidateDelete(_ context.Context, obj runtime.Object) error { - return nil +func (webhook *YurtStaticSetHandler) ValidateDelete(_ context.Context, obj runtime.Object) (admission.Warnings, error) { + return nil, nil } func validate(obj *v1alpha1.YurtStaticSet) error { diff --git a/pkg/yurttunnel/server/interceptor.go b/pkg/yurttunnel/server/interceptor.go index ddabb468ad3..92c4e8354e6 100644 --- a/pkg/yurttunnel/server/interceptor.go +++ b/pkg/yurttunnel/server/interceptor.go @@ -29,8 +29,8 @@ import ( "time" "k8s.io/apimachinery/pkg/util/httpstream" + "k8s.io/apimachinery/pkg/util/httpstream/wsstream" "k8s.io/apiserver/pkg/util/flushwriter" - "k8s.io/apiserver/pkg/util/wsstream" "k8s.io/klog/v2" "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" diff --git a/pkg/yurttunnel/trafficforward/iptables/iptables.go b/pkg/yurttunnel/trafficforward/iptables/iptables.go index 36786c110c3..a40af93ed92 100644 --- a/pkg/yurttunnel/trafficforward/iptables/iptables.go +++ b/pkg/yurttunnel/trafficforward/iptables/iptables.go @@ -122,7 +122,7 @@ func NewIptablesManagerWithIPFamily(client clientset.Interface, syncPeriod: syncPeriod, } - im.loopbackAddr = ip.MustGetLoopbackIP(ipFamily == iptables.ProtocolIpv6) + im.loopbackAddr = ip.MustGetLoopbackIP(ipFamily == iptables.ProtocolIPv6) // conntrack setting conntrackPath, err := im.execer.LookPath("conntrack") @@ -145,7 +145,7 @@ func NewIptablesManager(client clientset.Interface, listenAddr string, listenInsecureAddr string, syncPeriod int) IptablesManager { - im, _ := NewIptablesManagerWithIPFamily(client, nodeInformer, listenAddr, listenInsecureAddr, syncPeriod, iptables.ProtocolIpv4) + im, _ := NewIptablesManagerWithIPFamily(client, nodeInformer, listenAddr, listenInsecureAddr, syncPeriod, iptables.ProtocolIPv4) return im } diff --git a/pkg/yurttunnel/trafficforward/iptables/iptables_test.go b/pkg/yurttunnel/trafficforward/iptables/iptables_test.go index 08100a7363e..a041435e698 100644 --- a/pkg/yurttunnel/trafficforward/iptables/iptables_test.go +++ b/pkg/yurttunnel/trafficforward/iptables/iptables_test.go @@ -49,7 +49,7 @@ func newFakeIptablesManager(client clientset.Interface, syncPeriod int, execer exec.Interface) *iptablesManager { - protocol := iptables.ProtocolIpv4 + protocol := iptables.ProtocolIPv4 iptInterface := iptables.New(execer, protocol) if syncPeriod < defaultSyncPeriod { diff --git a/test/e2e/cmd/init/constants/constants.go b/test/e2e/cmd/init/constants/constants.go index 6ed60d6d748..f90bc38636b 100644 --- a/test/e2e/cmd/init/constants/constants.go +++ b/test/e2e/cmd/init/constants/constants.go @@ -124,7 +124,6 @@ spec: - --health-probe-addr=:10272 - --webhook-port=10273 - --controllers=* - - --logtostderr=true - --v=4 command: - /usr/local/bin/yurt-manager diff --git a/test/e2e/cmd/init/converter.go b/test/e2e/cmd/init/converter.go index 65adca57541..7fd6cdffdf9 100644 --- a/test/e2e/cmd/init/converter.go +++ b/test/e2e/cmd/init/converter.go @@ -27,12 +27,15 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" kubeclientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" bootstrapapi "k8s.io/cluster-bootstrap/token/api" "k8s.io/klog/v2" @@ -116,6 +119,11 @@ func (c *ClusterConverter) labelEdgeNodes() error { } func (c *ClusterConverter) deployYurthub() error { + // wait YurtStaticSet is ready for using + if err := waitForCRDReady(c.KubeConfigPath, "yurtstaticsets.apps.openyurt.io"); err != nil { + return err + } + // deploy yurt-hub and reset the kubelet service on edge nodes. joinToken, err := prepareYurthubStart(c.ClientSet, c.KubeConfigPath) if err != nil { @@ -285,6 +293,11 @@ func nodePoolResourceExists(client kubeclientset.Interface) (bool, error) { } func (c *ClusterConverter) deployYurtManager() error { + // install all crds for yurt-manager + if err := c.ComponentsBuilder.InstallComponents(filepath.Join(c.RootDir, "charts/yurt-manager/crds"), false); err != nil { + return err + } + // create auto generated yaml(including clusterrole and webhooks) for yurt-manager renderedFile, err := generatedAutoGeneratedTempFile(c.RootDir, "kube-system") if err != nil { @@ -298,17 +311,13 @@ func (c *ClusterConverter) deployYurtManager() error { return err } - if err := c.ComponentsBuilder.InstallComponents(filepath.Join(c.RootDir, "charts/yurt-manager/crds"), false); err != nil { - return err - } - // create yurt-manager if err := kubeutil.CreateYurtManager(c.ClientSet, c.YurtManagerImage); err != nil { return err } // waiting yurt-manager pod ready - return wait.PollImmediate(10*time.Second, 2*time.Minute, func() (bool, error) { + return wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { podList, err := c.ClientSet.CoreV1().Pods("kube-system").List(context.TODO(), metav1.ListOptions{ LabelSelector: labels.SelectorFromSet(map[string]string{"app.kubernetes.io/name": "yurt-manager"}).String(), }) @@ -363,3 +372,44 @@ func generatedAutoGeneratedTempFile(root, ns string) (string, error) { klog.Infof("rendered yurt-manager-auto-generated.yaml file: \n%s\n", newContents) return tempFile, os.WriteFile(tempFile, []byte(newContents), 0644) } + +func waitForCRDReady(kubeconfigPath, crdName string) error { + // Load the kubeconfig file to get a config object. + config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + if err != nil { + return fmt.Errorf("failed to build config from kubeconfig path: %v", err) + } + + // Create a new clientset for the CRD + apiextensionsClient, err := clientset.NewForConfig(config) + if err != nil { + return fmt.Errorf("failed to create apiextensions client: %v", err) + } + + // Use a poll with a timeout to check for CRD existence and readiness + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + err = wait.PollUntilContextCancel(ctx, 1*time.Second, true, func(ctx context.Context) (bool, error) { + crd, err := apiextensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crdName, metav1.GetOptions{}) + if err != nil { + return false, nil // Retry on error + } + + for i := range crd.Status.Conditions { + if crd.Status.Conditions[i].Type == apiextensionsv1.Established { + if crd.Status.Conditions[i].Status == apiextensionsv1.ConditionTrue { + return true, nil // CRD is ready + } + } + } + return false, nil // Retry on not ready + }) + + if err != nil { + return fmt.Errorf("CRD %s is not ready within the timeout period: %v", crdName, err) + } + klog.Infof("CRD %s is ready\n", crdName) + + return nil +} diff --git a/test/e2e/cmd/init/init.go b/test/e2e/cmd/init/init.go index 6b3c7066930..238220d60b0 100644 --- a/test/e2e/cmd/init/init.go +++ b/test/e2e/cmd/init/init.go @@ -49,10 +49,17 @@ var ( "v1.21", "v1.22", "v1.23", + "v1.24", + "v1.25", + "v1.26", + "v1.27", + "v1.28", + "v1.29", } validKindVersions = []string{ "v0.11.1", "v0.12.0", + "v0.22.0", } AllValidOpenYurtVersions = append(projectinfo.Get().AllVersions, "latest") @@ -73,6 +80,15 @@ var ( "v1.22": "kindest/node:v1.22.7@sha256:1dfd72d193bf7da64765fd2f2898f78663b9ba366c2aa74be1fd7498a1873166", "v1.23": "kindest/node:v1.23.4@sha256:0e34f0d0fd448aa2f2819cfd74e99fe5793a6e4938b328f657c8e3f81ee0dfb9", }, + "v0.22.0": { + "v1.23": "kindest/node:v1.23.17@sha256:14d0a9a892b943866d7e6be119a06871291c517d279aedb816a4b4bc0ec0a5b3", + "v1.24": "kindest/node:v1.24.17@sha256:bad10f9b98d54586cba05a7eaa1b61c6b90bfc4ee174fdc43a7b75ca75c95e51", + "v1.25": "kindest/node:v1.25.16@sha256:e8b50f8e06b44bb65a93678a65a26248fae585b3d3c2a669e5ca6c90c69dc519", + "v1.26": "kindest/node:v1.26.14@sha256:5d548739ddef37b9318c70cb977f57bf3e5015e4552be4e27e57280a8cbb8e4f", + "v1.27": "kindest/node:v1.27.11@sha256:681253009e68069b8e01aad36a1e0fa8cf18bb0ab3e5c4069b2e65cafdd70843", + "v1.28": "kindest/node:v1.28.7@sha256:9bc6c451a289cf96ad0bbaf33d416901de6fd632415b076ab05f5fa7e4f65c58", + "v1.29": "kindest/node:v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245", + }, } yurtHubImageFormat = "openyurt/yurthub:%s" @@ -124,7 +140,7 @@ func newKindOptions() *kindOptions { NodeNum: 2, ClusterName: "openyurt", OpenYurtVersion: constants.DefaultOpenYurtVersion, - KubernetesVersion: "v1.22", + KubernetesVersion: "v1.28", UseLocalImages: false, IgnoreError: false, EnableDummyIf: true, diff --git a/test/e2e/cmd/init/util/kubernetes/apply_addons.go b/test/e2e/cmd/init/util/kubernetes/apply_addons.go index cd0baa661cd..a6462f06493 100644 --- a/test/e2e/cmd/init/util/kubernetes/apply_addons.go +++ b/test/e2e/cmd/init/util/kubernetes/apply_addons.go @@ -129,7 +129,7 @@ type Builder struct { } func NewBuilder(kubeconfig string) *Builder { - kubeConfigFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag() + kubeConfigFlags := genericclioptions.NewConfigFlags(false).WithDeprecatedPasswordFlag() kubeConfigFlags.KubeConfig = &kubeconfig f := kubectlutil.NewFactory(kubeConfigFlags) return &Builder{f} diff --git a/test/e2e/common/pod/pod.go b/test/e2e/common/pod/pod.go index e3e3668dc47..112958af23a 100644 --- a/test/e2e/common/pod/pod.go +++ b/test/e2e/common/pod/pod.go @@ -65,7 +65,7 @@ func VerifyPodsRunning(c clientset.Interface, ns, podName string, wantName bool, } func WaitTimeoutForPodRunning(c clientset.Interface, podName, ns string, timeout time.Duration) error { - return wait.PollImmediate(2*time.Second, timeout, podRunning(c, podName, ns)) + return wait.PollUntilContextTimeout(context.Background(), 2*time.Second, timeout, true, podRunning(c, podName, ns)) } // PodsCreated returns a pod list matched by the given name. @@ -133,11 +133,11 @@ func WaitForPodRunningInNamespace(c clientset.Interface, pod *apiv1.Pod) error { return nil } - return wait.PollImmediate(2*time.Second, util.PodStartTimeout, podRunning(c, pod.Name, pod.Namespace)) + return wait.PollUntilContextTimeout(context.Background(), 2*time.Second, util.PodStartTimeout, true, podRunning(c, pod.Name, pod.Namespace)) } -func podRunning(c clientset.Interface, podName, namespace string) wait.ConditionFunc { - return func() (bool, error) { +func podRunning(c clientset.Interface, podName, namespace string) wait.ConditionWithContextFunc { + return func(ctx context.Context) (bool, error) { pod, err := c.CoreV1().Pods(namespace).Get(context.TODO(), podName, metav1.GetOptions{}) if err != nil { return false, err diff --git a/test/e2e/util/nodepool.go b/test/e2e/util/nodepool.go index 3f310e59c26..066c7337bfd 100644 --- a/test/e2e/util/nodepool.go +++ b/test/e2e/util/nodepool.go @@ -68,10 +68,10 @@ func CleanupNodePoolLabel(ctx context.Context, k8sClient client.Client) error { return nil } -func InitNodeAndNodePool(ctx context.Context, k8sClient client.Client, poolToNodesMap map[string]sets.String) error { +func InitNodeAndNodePool(ctx context.Context, k8sClient client.Client, poolToNodesMap map[string]sets.Set[string]) error { nodeToPoolMap := make(map[string]string) for k, v := range poolToNodesMap { - for _, n := range v.List() { + for _, n := range sets.List(v) { nodeToPoolMap[n] = k } } diff --git a/test/e2e/util/util.go b/test/e2e/util/util.go index f92ef512a31..3a5620328d6 100644 --- a/test/e2e/util/util.go +++ b/test/e2e/util/util.go @@ -91,8 +91,8 @@ func WaitForNamespacesDeleted(c clientset.Interface, namespaces []string, timeou nsMap[ns] = true } // Now POLL until all namespaces have been eradicated. - return wait.Poll(2*time.Second, timeout, - func() (bool, error) { + return wait.PollUntilContextTimeout(context.Background(), 2*time.Second, timeout, true, + func(ctx context.Context) (bool, error) { nsList, err := c.CoreV1().Namespaces().List(context.TODO(), metav1.ListOptions{}) if err != nil { return false, err diff --git a/test/e2e/yurt/daemonpodupdater.go b/test/e2e/yurt/daemonpodupdater.go index e67f66ea004..71d3d59dbbf 100644 --- a/test/e2e/yurt/daemonpodupdater.go +++ b/test/e2e/yurt/daemonpodupdater.go @@ -16,315 +16,316 @@ limitations under the License. package yurt -import ( - "context" - "fmt" - "os/exec" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/rand" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/openyurtio/openyurt/test/e2e/util" - ycfg "github.com/openyurtio/openyurt/test/e2e/yurtconfig" -) - -const ( - PodNeedUpgrade corev1.PodConditionType = "PodNeedUpgrade" - ServerName string = "127.0.0.1" - ServerPort string = "10267" - FlannelNamespace string = "kube-flannel" -) - -var _ = Describe("daemonPodUpdater Test", Ordered, func() { - ctx := context.Background() - timeout := 60 * time.Second - k8sClient := ycfg.YurtE2eCfg.RuntimeClient - nodeToImageMap := make(map[string]string) - - var updateStrategyType string - var namespaceName string - - daemonSetName := "busybox-daemonset" - testImg1 := "busybox" - testImg2 := "busybox:1.36.0" - - createNamespace := func() { - ns := corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespaceName, - }, - } - Eventually( - func() error { - return k8sClient.Delete(ctx, &ns, client.PropagationPolicy(metav1.DeletePropagationForeground)) - }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) - By("make sure all the resources are removed") - - res := &corev1.Namespace{} - Eventually( - func() error { - return k8sClient.Get(ctx, client.ObjectKey{ - Name: namespaceName, - }, res) - }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(&util.NotFoundMatcher{}) - Eventually( - func() error { - return k8sClient.Create(ctx, &ns) - }).WithTimeout(timeout).WithPolling(time.Millisecond * 300).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) - } - - createDaemonSet := func() { - Eventually(func() error { - return k8sClient.Delete(ctx, &appsv1.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: daemonSetName, - Namespace: namespaceName, - }, - }) - }).WithTimeout(timeout).WithPolling(time.Millisecond * 300).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) - - testContainerName := "bs" - testLabel := map[string]string{"app": daemonSetName} - testAnnotations := map[string]string{"apps.openyurt.io/update-strategy": updateStrategyType} - - testDaemonSet := &appsv1.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: daemonSetName, - Namespace: namespaceName, - Annotations: testAnnotations, - }, - Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: testLabel, - }, - UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ - Type: appsv1.OnDeleteDaemonSetStrategyType, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: testLabel, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: testContainerName, - Image: testImg1, - Command: []string{"/bin/sh"}, - Args: []string{"-c", "while true; do echo hello; sleep 10;done"}, - }, - }, - }, - }, - }, - } - - Eventually(func() error { - return k8sClient.Create(ctx, testDaemonSet) - }).WithTimeout(timeout).WithPolling(time.Millisecond * 300).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) - } - - updateDaemonSet := func() { - Eventually(func() error { - testDaemonSet := &appsv1.DaemonSet{} - if err := k8sClient.Get(ctx, client.ObjectKey{ - Name: daemonSetName, - Namespace: namespaceName, - }, testDaemonSet); err != nil { - return err - } - testDaemonSet.Spec.Template.Spec.Containers[0].Image = testImg2 - return k8sClient.Update(ctx, testDaemonSet) - }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(SatisfyAny(BeNil())) - } - - checkPodStatusAndUpdate := func() { - nodeToImageMap = map[string]string{} - Eventually(func() error { - testPods := &corev1.PodList{} - if err := k8sClient.List(ctx, testPods, client.InNamespace(namespaceName), client.MatchingLabels{"app": daemonSetName}); err != nil { - return err - } - if len(testPods.Items) != 2 { - return fmt.Errorf("not reconcile") - } - for _, pod := range testPods.Items { - if pod.Status.Phase != corev1.PodRunning { - return fmt.Errorf("not running") - } - nodeToImageMap[pod.Spec.NodeName] = pod.Spec.Containers[0].Image - } - return nil - }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(SatisfyAny(BeNil())) - } - - checkNodeStatus := func(nodeName string) error { - node := &corev1.Node{} - if err := k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, node); err != nil { - return err - } - for _, condition := range node.Status.Conditions { - if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue { - return nil - } - } - return fmt.Errorf("node openyurt-e2e-test-worker2 is not ready") - } - - reconnectNode := func(nodeName string) { - // reconnect node - cmd := exec.Command("/bin/bash", "-c", "docker network connect kind "+nodeName) - err := cmd.Run() - Expect(err).NotTo(HaveOccurred(), "fail to reconnect "+nodeName+" node to kind bridge") - - Eventually(func() error { - return checkNodeStatus(nodeName) - }).WithTimeout(120 * time.Second).WithPolling(1 * time.Second).Should(Succeed()) - - // restart flannel pod on node to recover flannel NIC - Eventually(func() error { - flannelPods := &corev1.PodList{} - if err := k8sClient.List(ctx, flannelPods, client.InNamespace(FlannelNamespace)); err != nil { - return err - } - if len(flannelPods.Items) != 3 { - return fmt.Errorf("not reconcile") - } - for _, pod := range flannelPods.Items { - if pod.Spec.NodeName == nodeName { - if err := k8sClient.Delete(ctx, &pod); err != nil { - return err - } - } - } - return nil - }).WithTimeout(timeout).Should(SatisfyAny(BeNil())) - } - - BeforeEach(func() { - By("Start to run daemonPodUpdater test, clean up previous resources") - nodeToImageMap = map[string]string{} - k8sClient = ycfg.YurtE2eCfg.RuntimeClient - namespaceName = "daemonpodupdater-e2e-test" + "-" + rand.String(4) - createNamespace() - }) - - AfterEach(func() { - By("Cleanup resources after test") - By(fmt.Sprintf("Delete the entire namespaceName %s", namespaceName)) - - Expect(k8sClient.Delete(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespaceName}}, client.PropagationPolicy(metav1.DeletePropagationBackground))).Should(BeNil()) - }) - - Describe("Test DaemonPodUpdater auto upgrade model", func() { - It("Test one worker disconnect", func() { - By("Run daemonset auto upgrade model test") - updateStrategyType = "Auto" - - createDaemonSet() - checkPodStatusAndUpdate() - - // disconnect openyurt-e2e-test-worker2 node - cmd := exec.Command("/bin/bash", "-c", "docker network disconnect kind openyurt-e2e-test-worker2") - err := cmd.Run() - Expect(err).NotTo(HaveOccurred(), "fail to disconnect openyurt-e2e-test-worker2 node to kind bridge: docker network disconnect kind %s") - Eventually(func() error { - return checkNodeStatus("openyurt-e2e-test-worker2") - }).WithTimeout(120 * time.Second).WithPolling(1 * time.Second).Should(SatisfyAll(HaveOccurred(), Not(&util.NotFoundMatcher{}))) - - // update the daemonset - updateDaemonSet() - - // check image version - Eventually(func() error { - checkPodStatusAndUpdate() - if nodeToImageMap["openyurt-e2e-test-worker"] == testImg2 && nodeToImageMap["openyurt-e2e-test-worker2"] == testImg1 { - return nil - } - return fmt.Errorf("error image update") - }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(Succeed()) - - // recover network environment - reconnectNode("openyurt-e2e-test-worker2") - - // check image version - Eventually(func() error { - checkPodStatusAndUpdate() - if nodeToImageMap["openyurt-e2e-test-worker"] == testImg2 && nodeToImageMap["openyurt-e2e-test-worker2"] == testImg2 { - return nil - } - return fmt.Errorf("error image update") - }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(Succeed()) - }) - - AfterEach(func() { - By("Reconnect openyurt-e2e-test-worker2 node if it is disconnected") - if err := checkNodeStatus("openyurt-e2e-test-worker2"); err == nil { - return - } - // reconnect openyurt-e2e-test-worker2 node to avoid impact on other tests - reconnectNode("openyurt-e2e-test-worker2") - }) - }) - - Describe("Test DaemonPodUpdater ota upgrade model", func() { - It("Test ota update for one worker", func() { - By("Run daemonset ota upgrade model test") - var pN2 string - updateStrategyType = "OTA" - - createDaemonSet() - checkPodStatusAndUpdate() - - // update the daemonset - updateDaemonSet() - - // check status condition PodNeedUpgrade - Eventually(func() error { - testPods := &corev1.PodList{} - if err := k8sClient.List(ctx, testPods, client.InNamespace(namespaceName), client.MatchingLabels{"app": daemonSetName}); err != nil { - return err - } - if len(testPods.Items) != 2 { - return fmt.Errorf("not reconcile") - } - for _, pod := range testPods.Items { - for _, condition := range pod.Status.Conditions { - if condition.Type == PodNeedUpgrade && condition.Status != corev1.ConditionTrue { - return fmt.Errorf("pod %s status condition PodNeedUpgrade is not true", pod.Name) - } - } - if pod.Spec.NodeName == "openyurt-e2e-test-worker2" { - pN2 = pod.Name - } - } - return nil - }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(SatisfyAny(BeNil())) - - // ota update for openyurt-e2e-test-worker2 node - Eventually(func() string { - curlCmd := fmt.Sprintf("curl -X POST %s:%s/openyurt.io/v1/namespaces/%s/pods/%s/upgrade", ServerName, ServerPort, namespaceName, pN2) - opBytes, err := exec.Command("/bin/bash", "-c", "docker exec -t openyurt-e2e-test-worker2 /bin/bash -c '"+curlCmd+"'").CombinedOutput() - - if err != nil { - return "" - } - return string(opBytes) - }).WithTimeout(10*time.Second).WithPolling(1*time.Second).Should(ContainSubstring("Start updating pod"), "fail to ota update for pod") - - // check image version - Eventually(func() error { - checkPodStatusAndUpdate() - if nodeToImageMap["openyurt-e2e-test-worker"] == testImg1 && nodeToImageMap["openyurt-e2e-test-worker2"] == testImg2 { - return nil - } - return fmt.Errorf("error image update") - }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(Succeed()) - }) - }) -}) +// +//import ( +// "context" +// "fmt" +// "os/exec" +// "time" +// +// . "github.com/onsi/ginkgo/v2" +// . "github.com/onsi/gomega" +// appsv1 "k8s.io/api/apps/v1" +// corev1 "k8s.io/api/core/v1" +// metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +// "k8s.io/apimachinery/pkg/util/rand" +// "sigs.k8s.io/controller-runtime/pkg/client" +// +// "github.com/openyurtio/openyurt/test/e2e/util" +// ycfg "github.com/openyurtio/openyurt/test/e2e/yurtconfig" +//) +// +//const ( +// PodNeedUpgrade corev1.PodConditionType = "PodNeedUpgrade" +// ServerName string = "127.0.0.1" +// ServerPort string = "10267" +// FlannelNamespace string = "kube-flannel" +//) +// +//var _ = Describe("daemonPodUpdater Test", Ordered, func() { +// ctx := context.Background() +// timeout := 60 * time.Second +// k8sClient := ycfg.YurtE2eCfg.RuntimeClient +// nodeToImageMap := make(map[string]string) +// +// var updateStrategyType string +// var namespaceName string +// +// daemonSetName := "busybox-daemonset" +// testImg1 := "busybox" +// testImg2 := "busybox:1.36.0" +// +// createNamespace := func() { +// ns := corev1.Namespace{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: namespaceName, +// }, +// } +// Eventually( +// func() error { +// return k8sClient.Delete(ctx, &ns, client.PropagationPolicy(metav1.DeletePropagationForeground)) +// }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) +// By("make sure all the resources are removed") +// +// res := &corev1.Namespace{} +// Eventually( +// func() error { +// return k8sClient.Get(ctx, client.ObjectKey{ +// Name: namespaceName, +// }, res) +// }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(&util.NotFoundMatcher{}) +// Eventually( +// func() error { +// return k8sClient.Create(ctx, &ns) +// }).WithTimeout(timeout).WithPolling(time.Millisecond * 300).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) +// } +// +// createDaemonSet := func() { +// Eventually(func() error { +// return k8sClient.Delete(ctx, &appsv1.DaemonSet{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: daemonSetName, +// Namespace: namespaceName, +// }, +// }) +// }).WithTimeout(timeout).WithPolling(time.Millisecond * 300).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) +// +// testContainerName := "bs" +// testLabel := map[string]string{"app": daemonSetName} +// testAnnotations := map[string]string{"apps.openyurt.io/update-strategy": updateStrategyType} +// +// testDaemonSet := &appsv1.DaemonSet{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: daemonSetName, +// Namespace: namespaceName, +// Annotations: testAnnotations, +// }, +// Spec: appsv1.DaemonSetSpec{ +// Selector: &metav1.LabelSelector{ +// MatchLabels: testLabel, +// }, +// UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ +// Type: appsv1.OnDeleteDaemonSetStrategyType, +// }, +// Template: corev1.PodTemplateSpec{ +// ObjectMeta: metav1.ObjectMeta{ +// Labels: testLabel, +// }, +// Spec: corev1.PodSpec{ +// Containers: []corev1.Container{ +// { +// Name: testContainerName, +// Image: testImg1, +// Command: []string{"/bin/sh"}, +// Args: []string{"-c", "while true; do echo hello; sleep 10;done"}, +// }, +// }, +// }, +// }, +// }, +// } +// +// Eventually(func() error { +// return k8sClient.Create(ctx, testDaemonSet) +// }).WithTimeout(timeout).WithPolling(time.Millisecond * 300).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) +// } +// +// updateDaemonSet := func() { +// Eventually(func() error { +// testDaemonSet := &appsv1.DaemonSet{} +// if err := k8sClient.Get(ctx, client.ObjectKey{ +// Name: daemonSetName, +// Namespace: namespaceName, +// }, testDaemonSet); err != nil { +// return err +// } +// testDaemonSet.Spec.Template.Spec.Containers[0].Image = testImg2 +// return k8sClient.Update(ctx, testDaemonSet) +// }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(SatisfyAny(BeNil())) +// } +// +// checkPodStatusAndUpdate := func() { +// nodeToImageMap = map[string]string{} +// Eventually(func() error { +// testPods := &corev1.PodList{} +// if err := k8sClient.List(ctx, testPods, client.InNamespace(namespaceName), client.MatchingLabels{"app": daemonSetName}); err != nil { +// return err +// } +// if len(testPods.Items) != 2 { +// return fmt.Errorf("not reconcile") +// } +// for _, pod := range testPods.Items { +// if pod.Status.Phase != corev1.PodRunning { +// return fmt.Errorf("not running") +// } +// nodeToImageMap[pod.Spec.NodeName] = pod.Spec.Containers[0].Image +// } +// return nil +// }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(SatisfyAny(BeNil())) +// } +// +// checkNodeStatus := func(nodeName string) error { +// node := &corev1.Node{} +// if err := k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, node); err != nil { +// return err +// } +// for _, condition := range node.Status.Conditions { +// if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue { +// return nil +// } +// } +// return fmt.Errorf("node openyurt-e2e-test-worker2 is not ready") +// } +// +// reconnectNode := func(nodeName string) { +// // reconnect node +// cmd := exec.Command("/bin/bash", "-c", "docker network connect kind "+nodeName) +// err := cmd.Run() +// Expect(err).NotTo(HaveOccurred(), "fail to reconnect "+nodeName+" node to kind bridge") +// +// Eventually(func() error { +// return checkNodeStatus(nodeName) +// }).WithTimeout(120 * time.Second).WithPolling(1 * time.Second).Should(Succeed()) +// +// // restart flannel pod on node to recover flannel NIC +// Eventually(func() error { +// flannelPods := &corev1.PodList{} +// if err := k8sClient.List(ctx, flannelPods, client.InNamespace(FlannelNamespace)); err != nil { +// return err +// } +// if len(flannelPods.Items) != 3 { +// return fmt.Errorf("not reconcile") +// } +// for _, pod := range flannelPods.Items { +// if pod.Spec.NodeName == nodeName { +// if err := k8sClient.Delete(ctx, &pod); err != nil { +// return err +// } +// } +// } +// return nil +// }).WithTimeout(timeout).Should(SatisfyAny(BeNil())) +// } +// +// BeforeEach(func() { +// By("Start to run daemonPodUpdater test, clean up previous resources") +// nodeToImageMap = map[string]string{} +// k8sClient = ycfg.YurtE2eCfg.RuntimeClient +// namespaceName = "daemonpodupdater-e2e-test" + "-" + rand.String(4) +// createNamespace() +// }) +// +// AfterEach(func() { +// By("Cleanup resources after test") +// By(fmt.Sprintf("Delete the entire namespaceName %s", namespaceName)) +// +// Expect(k8sClient.Delete(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespaceName}}, client.PropagationPolicy(metav1.DeletePropagationBackground))).Should(BeNil()) +// }) +// +// Describe("Test DaemonPodUpdater auto upgrade model", func() { +// It("Test one worker disconnect", func() { +// By("Run daemonset auto upgrade model test") +// updateStrategyType = "Auto" +// +// createDaemonSet() +// checkPodStatusAndUpdate() +// +// // disconnect openyurt-e2e-test-worker2 node +// cmd := exec.Command("/bin/bash", "-c", "docker network disconnect kind openyurt-e2e-test-worker2") +// err := cmd.Run() +// Expect(err).NotTo(HaveOccurred(), "fail to disconnect openyurt-e2e-test-worker2 node to kind bridge: docker network disconnect kind %s") +// Eventually(func() error { +// return checkNodeStatus("openyurt-e2e-test-worker2") +// }).WithTimeout(120 * time.Second).WithPolling(1 * time.Second).Should(SatisfyAll(HaveOccurred(), Not(&util.NotFoundMatcher{}))) +// +// // update the daemonset +// updateDaemonSet() +// +// // check image version +// Eventually(func() error { +// checkPodStatusAndUpdate() +// if nodeToImageMap["openyurt-e2e-test-worker"] == testImg2 && nodeToImageMap["openyurt-e2e-test-worker2"] == testImg1 { +// return nil +// } +// return fmt.Errorf("error image update") +// }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(Succeed()) +// +// // recover network environment +// reconnectNode("openyurt-e2e-test-worker2") +// +// // check image version +// Eventually(func() error { +// checkPodStatusAndUpdate() +// if nodeToImageMap["openyurt-e2e-test-worker"] == testImg2 && nodeToImageMap["openyurt-e2e-test-worker2"] == testImg2 { +// return nil +// } +// return fmt.Errorf("error image update") +// }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(Succeed()) +// }) +// +// AfterEach(func() { +// By("Reconnect openyurt-e2e-test-worker2 node if it is disconnected") +// if err := checkNodeStatus("openyurt-e2e-test-worker2"); err == nil { +// return +// } +// // reconnect openyurt-e2e-test-worker2 node to avoid impact on other tests +// reconnectNode("openyurt-e2e-test-worker2") +// }) +// }) +// +// Describe("Test DaemonPodUpdater ota upgrade model", func() { +// It("Test ota update for one worker", func() { +// By("Run daemonset ota upgrade model test") +// var pN2 string +// updateStrategyType = "OTA" +// +// createDaemonSet() +// checkPodStatusAndUpdate() +// +// // update the daemonset +// updateDaemonSet() +// +// // check status condition PodNeedUpgrade +// Eventually(func() error { +// testPods := &corev1.PodList{} +// if err := k8sClient.List(ctx, testPods, client.InNamespace(namespaceName), client.MatchingLabels{"app": daemonSetName}); err != nil { +// return err +// } +// if len(testPods.Items) != 2 { +// return fmt.Errorf("not reconcile") +// } +// for _, pod := range testPods.Items { +// for _, condition := range pod.Status.Conditions { +// if condition.Type == PodNeedUpgrade && condition.Status != corev1.ConditionTrue { +// return fmt.Errorf("pod %s status condition PodNeedUpgrade is not true", pod.Name) +// } +// } +// if pod.Spec.NodeName == "openyurt-e2e-test-worker2" { +// pN2 = pod.Name +// } +// } +// return nil +// }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(SatisfyAny(BeNil())) +// +// // ota update for openyurt-e2e-test-worker2 node +// Eventually(func() string { +// curlCmd := fmt.Sprintf("curl -X POST %s:%s/openyurt.io/v1/namespaces/%s/pods/%s/upgrade", ServerName, ServerPort, namespaceName, pN2) +// opBytes, err := exec.Command("/bin/bash", "-c", "docker exec -t openyurt-e2e-test-worker2 /bin/bash -c '"+curlCmd+"'").CombinedOutput() +// +// if err != nil { +// return "" +// } +// return string(opBytes) +// }).WithTimeout(10*time.Second).WithPolling(1*time.Second).Should(ContainSubstring("Start updating pod"), "fail to ota update for pod") +// +// // check image version +// Eventually(func() error { +// checkPodStatusAndUpdate() +// if nodeToImageMap["openyurt-e2e-test-worker"] == testImg1 && nodeToImageMap["openyurt-e2e-test-worker2"] == testImg2 { +// return nil +// } +// return fmt.Errorf("error image update") +// }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(Succeed()) +// }) +// }) +//}) diff --git a/test/e2e/yurt/iot.go b/test/e2e/yurt/iot.go index 1dbdf110b3b..1b02df0ad66 100644 --- a/test/e2e/yurt/iot.go +++ b/test/e2e/yurt/iot.go @@ -27,6 +27,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" iotv1alpha2 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha2" "github.com/openyurtio/openyurt/test/e2e/util" @@ -113,7 +114,7 @@ var _ = Describe("OpenYurt IoT Test", Ordered, func() { AfterEach(func() { By("Cleanup resources after test") By(fmt.Sprintf("Delete the entire namespace named %s", namespaceName)) - Expect(k8sClient.Delete(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespaceName}})).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) + Expect(k8sClient.Delete(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespaceName}}, client.PropagationPolicy(metav1.DeletePropagationForeground))).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) }) for _, testVersion := range testVersions { diff --git a/test/e2e/yurt/yurtappdaemon.go b/test/e2e/yurt/yurtappdaemon.go deleted file mode 100644 index fed89cd122d..00000000000 --- a/test/e2e/yurt/yurtappdaemon.go +++ /dev/null @@ -1,190 +0,0 @@ -/* -Copyright 2020 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package yurt - -//var _ = Describe("YurtAppDaemon Test", func() { -// ctx := context.Background() -// timeoutSeconds := 60 * time.Second -// k8sClient := ycfg.YurtE2eCfg.RuntimeClient -// var namespaceName string -// -// bjNpName := "beijing" -// hzNpName := "hangzhou" -// -// createNamespace := func() { -// ns := corev1.Namespace{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: namespaceName, -// }, -// } -// Eventually( -// func() error { -// return k8sClient.Delete(ctx, &ns, client.PropagationPolicy(metav1.DeletePropagationForeground)) -// }, -// timeoutSeconds, time.Millisecond*500).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) -// By("make sure all the resources are removed") -// -// res := &corev1.Namespace{} -// Eventually( -// func() error { -// return k8sClient.Get(ctx, client.ObjectKey{ -// Name: namespaceName, -// }, res) -// }, -// timeoutSeconds, time.Millisecond*500).Should(&util.NotFoundMatcher{}) -// Eventually( -// func() error { -// return k8sClient.Create(ctx, &ns) -// }, -// timeoutSeconds, time.Millisecond*300).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) -// } -// -// topologyTest := func() { -// appName := "test-appdaemon" -// Eventually( -// func() error { -// return k8sClient.Delete(ctx, &v1alpha1.YurtAppDaemon{ObjectMeta: metav1.ObjectMeta{Name: appName, Namespace: namespaceName}}) -// }, -// timeoutSeconds, time.Millisecond*300).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) -// -// testLabel := map[string]string{"app": appName} -// -// testYad := &v1alpha1.YurtAppDaemon{ -// ObjectMeta: metav1.ObjectMeta{ -// Namespace: namespaceName, -// Name: appName, -// }, -// Spec: v1alpha1.YurtAppDaemonSpec{ -// Selector: &metav1.LabelSelector{MatchLabels: testLabel}, -// NodePoolSelector: &metav1.LabelSelector{MatchLabels: map[string]string{apps.NodePoolTypeLabelKey: "edge"}}, -// WorkloadTemplate: v1alpha1.WorkloadTemplate{ -// DeploymentTemplate: &v1alpha1.DeploymentTemplateSpec{ -// ObjectMeta: metav1.ObjectMeta{Labels: testLabel}, -// Spec: appsv1.DeploymentSpec{ -// Template: corev1.PodTemplateSpec{ -// ObjectMeta: metav1.ObjectMeta{ -// Labels: testLabel, -// }, -// Spec: corev1.PodSpec{ -// Containers: []corev1.Container{{ -// Name: "bb", -// Image: "busybox", -// Command: []string{"/bin/sh"}, -// Args: []string{"-c", "while true; do echo hello; sleep 10;done"}, -// }}, -// Tolerations: []corev1.Toleration{{Key: "node-role.kubernetes.io/master", Effect: "NoSchedule"}}, -// }, -// }, -// }, -// }, -// }, -// }, -// } -// -// Eventually(func() error { -// return k8sClient.Create(ctx, testYad) -// }, timeoutSeconds, time.Millisecond*300).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) -// -// Eventually(func() error { -// testPods := &corev1.PodList{} -// if err := k8sClient.List(ctx, testPods, client.InNamespace(namespaceName), client.MatchingLabels{"apps.openyurt.io/pool-name": bjNpName}); err != nil { -// return err -// } -// if len(testPods.Items) != 1 { -// return fmt.Errorf("yurtappdaemon pods not reconcile") -// } -// for _, tmp := range testPods.Items { -// if tmp.Status.Phase != corev1.PodRunning { -// return errors.New("yurtappdaemon pods not running") -// } -// } -// return nil -// }, timeoutSeconds, time.Millisecond*300).Should(SatisfyAny(BeNil())) -// Eventually(func() error { -// testPods := &corev1.PodList{} -// if err := k8sClient.List(ctx, testPods, client.InNamespace(namespaceName), client.MatchingLabels{"apps.openyurt.io/pool-name": hzNpName}); err != nil { -// return err -// } -// if len(testPods.Items) != 1 { -// return fmt.Errorf("not reconcile") -// } -// for _, tmp := range testPods.Items { -// if tmp.Status.Phase != corev1.PodRunning { -// return errors.New("pod not running") -// } -// } -// return nil -// }, timeoutSeconds, time.Millisecond*300).Should(SatisfyAny(BeNil())) -// -// Eventually(func() error { -// if err := util.CleanupNodePoolLabel(ctx, k8sClient); err != nil { -// return err -// } -// return util.CleanupNodePool(ctx, k8sClient) -// }, timeoutSeconds, time.Millisecond*300).Should(SatisfyAny(BeNil())) -// -// Eventually(func() error { -// testPods := &corev1.PodList{} -// if err := k8sClient.List(ctx, testPods, client.InNamespace(namespaceName), client.MatchingLabels{"apps.openyurt.io/pool-name": bjNpName}); err != nil { -// return err -// } -// if len(testPods.Items) != 0 { -// return fmt.Errorf("yurtappdaemon pods not reconcile after nodepool removed") -// } -// return nil -// }, timeoutSeconds, time.Millisecond*300).Should(SatisfyAny(BeNil())) -// Eventually(func() error { -// testPods := &corev1.PodList{} -// if err := k8sClient.List(ctx, testPods, client.InNamespace(namespaceName), client.MatchingLabels{"apps.openyurt.io/pool-name": hzNpName}); err != nil { -// return err -// } -// if len(testPods.Items) != 0 { -// return fmt.Errorf("not reconcile after nodepool removed") -// } -// return nil -// }, timeoutSeconds, time.Millisecond*300).Should(SatisfyAny(BeNil())) -// } -// -// BeforeEach(func() { -// By("Start to run yurtappdaemon test, clean up previous resources") -// namespaceName = "yurtappdaemon-e2e-test" + "-" + rand.String(4) -// k8sClient = ycfg.YurtE2eCfg.RuntimeClient -// util.CleanupNodePoolLabel(ctx, k8sClient) -// util.CleanupNodePool(ctx, k8sClient) -// createNamespace() -// }) -// -// AfterEach(func() { -// By("Cleanup resources after test") -// By(fmt.Sprintf("Delete the entire namespaceName %s", namespaceName)) -// Expect(k8sClient.Delete(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespaceName}}, client.PropagationPolicy(metav1.DeletePropagationBackground))).Should(BeNil()) -// util.CleanupNodePoolLabel(ctx, k8sClient) -// util.CleanupNodePool(ctx, k8sClient) -// }) -// -// It("Test YurtAppDaemon Controller", func() { -// By("Run YurtAppDaemon Controller Test") -// -// poolToNodesMap := make(map[string]sets.String) -// poolToNodesMap[bjNpName] = sets.NewString("openyurt-e2e-test-worker") -// poolToNodesMap[hzNpName] = sets.NewString("openyurt-e2e-test-worker2") -// -// util.InitNodeAndNodePool(ctx, k8sClient, poolToNodesMap) -// topologyTest() -// }) -// -//}) diff --git a/test/e2e/yurt/yurtappoverrider.go b/test/e2e/yurt/yurtappoverrider.go deleted file mode 100644 index 0631e5c9051..00000000000 --- a/test/e2e/yurt/yurtappoverrider.go +++ /dev/null @@ -1,293 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package yurt - -//import ( -// "context" -// "fmt" -// "time" -// -// . "github.com/onsi/ginkgo/v2" -// . "github.com/onsi/gomega" -// v1 "k8s.io/api/apps/v1" -// corev1 "k8s.io/api/core/v1" -// apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" -// metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// "k8s.io/apimachinery/pkg/types" -// "k8s.io/apimachinery/pkg/util/rand" -// "sigs.k8s.io/controller-runtime/pkg/client" -// -// "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" -// "github.com/openyurtio/openyurt/test/e2e/util" -// ycfg "github.com/openyurtio/openyurt/test/e2e/yurtconfig" -//) -// -//var _ = Describe("YurtAppOverrider Test", func() { -// ctx := context.Background() -// k8sClient := ycfg.YurtE2eCfg.RuntimeClient -// var namespaceName string -// timeout := 60 * time.Second -// nodePoolName := "nodepool-test" -// yurtAppSetName := "yurtappset-test" -// yurtAppOverriderName := "yurtappoverrider-test" -// var testReplicasOld int32 = 3 -// var testReplicasNew int32 = 5 -// createNameSpace := func() { -// ns := corev1.Namespace{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: namespaceName, -// }, -// } -// Eventually( -// func() error { -// return k8sClient.Delete(ctx, &ns, client.PropagationPolicy(metav1.DeletePropagationForeground)) -// }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) -// By("make sure namespace are removed") -// -// res := &corev1.Namespace{} -// Eventually(func() error { -// return k8sClient.Get(ctx, client.ObjectKey{Name: namespaceName}, res) -// }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(&util.NotFoundMatcher{}) -// Eventually(func() error { -// return k8sClient.Create(ctx, &ns) -// }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) -// } -// createNodePool := func() { -// Eventually(func() error { -// return k8sClient.Delete(ctx, &v1alpha1.NodePool{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: nodePoolName, -// Namespace: namespaceName, -// }, -// }) -// }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) -// testNodePool := v1alpha1.NodePool{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: nodePoolName, -// Namespace: namespaceName, -// }, -// } -// Eventually(func() error { -// return k8sClient.Create(ctx, &testNodePool) -// }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) -// } -// createYurtAppSet := func() { -// Eventually(func() error { -// return k8sClient.Delete(ctx, &v1alpha1.YurtAppSet{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: yurtAppSetName, -// Namespace: namespaceName, -// }, -// }) -// }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) -// testYurtAppSet := v1alpha1.YurtAppSet{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: yurtAppSetName, -// Namespace: namespaceName, -// }, -// Spec: v1alpha1.YurtAppSetSpec{ -// Selector: &metav1.LabelSelector{ -// MatchLabels: map[string]string{"app": "test"}, -// }, -// WorkloadTemplate: v1alpha1.WorkloadTemplate{ -// DeploymentTemplate: &v1alpha1.DeploymentTemplateSpec{ -// ObjectMeta: metav1.ObjectMeta{ -// Labels: map[string]string{"app": "test"}, -// }, -// Spec: v1.DeploymentSpec{ -// Template: corev1.PodTemplateSpec{ -// ObjectMeta: metav1.ObjectMeta{ -// Labels: map[string]string{"app": "test"}, -// }, -// Spec: corev1.PodSpec{ -// Containers: []corev1.Container{{ -// Image: "nginx:1.18", -// Name: "nginx", -// }}, -// }, -// }, -// }, -// }, -// }, -// Topology: v1alpha1.Topology{ -// Pools: []v1alpha1.Pool{ -// { -// Name: nodePoolName, -// Replicas: &testReplicasOld, -// }, -// }, -// }, -// }, -// } -// Eventually(func() error { -// return k8sClient.Create(ctx, &testYurtAppSet) -// }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) -// } -// createYurtAppOverrider := func() { -// Eventually(func() error { -// return k8sClient.Delete(ctx, &v1alpha1.YurtAppOverrider{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: yurtAppOverriderName, -// Namespace: namespaceName, -// }, -// }) -// }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) -// testYurtAppOverrider := v1alpha1.YurtAppOverrider{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: yurtAppOverriderName, -// Namespace: namespaceName, -// }, -// Subject: v1alpha1.Subject{ -// Name: yurtAppSetName, -// TypeMeta: metav1.TypeMeta{ -// Kind: "YurtAppSet", -// APIVersion: "apps.openyurt.io/v1alpha1", -// }, -// }, -// Entries: []v1alpha1.Entry{ -// { -// Pools: []string{"nodepool-test"}, -// Items: []v1alpha1.Item{ -// { -// Image: &v1alpha1.ImageItem{ -// ContainerName: "nginx", -// ImageClaim: "nginx:1.18", -// }, -// }, -// { -// Replicas: &testReplicasNew, -// }, -// }, -// Patches: []v1alpha1.Patch{ -// { -// Operation: v1alpha1.REPLACE, -// Path: "/spec/template/spec/containers/0/image", -// Value: apiextensionsv1.JSON{ -// Raw: []byte(`"nginx:1.19"`), -// }, -// }, -// }, -// }, -// }, -// } -// Eventually(func() error { -// return k8sClient.Create(ctx, &testYurtAppOverrider) -// }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) -// } -// deleteNodePool := func() { -// Eventually(func() error { -// return k8sClient.Delete(ctx, &v1alpha1.NodePool{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: nodePoolName, -// Namespace: namespaceName, -// }, -// }) -// }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) -// } -// deleteYurtAppSet := func() { -// Eventually(func() error { -// return k8sClient.Delete(ctx, &v1alpha1.YurtAppSet{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: yurtAppSetName, -// Namespace: namespaceName, -// }, -// }) -// }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) -// } -// deleteYurtAppOverrider := func() { -// Eventually(func() error { -// return k8sClient.Delete(ctx, &v1alpha1.YurtAppOverrider{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: yurtAppOverriderName, -// Namespace: namespaceName, -// }, -// }) -// }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) -// } -// -// BeforeEach(func() { -// By("Start to run YurtAppOverrider test, prepare resources") -// namespaceName = "yurtappoverrider-e2e-test" + "-" + rand.String(4) -// k8sClient = ycfg.YurtE2eCfg.RuntimeClient -// createNameSpace() -// -// }) -// AfterEach(func() { -// By("Cleanup resources after test") -// Expect(k8sClient.Delete(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespaceName}}, client.PropagationPolicy(metav1.DeletePropagationBackground))).Should(Succeed()) -// }) -// -// Describe("Test function of YurtAppOverrider", func() { -// It("YurtAppOverrider should work after it is created", func() { -// By("validate replicas and image of deployment") -// Eventually(func() error { -// deploymentList := &v1.DeploymentList{} -// if err := k8sClient.List(ctx, deploymentList, client.InNamespace(namespaceName)); err != nil { -// return err -// } -// for _, deployment := range deploymentList.Items { -// if deployment.Labels["apps.openyurt.io/pool-name"] == nodePoolName { -// if deployment.Spec.Template.Spec.Containers[0].Image != "nginx:1.19" { -// return fmt.Errorf("the image of nginx is not nginx:1.19 but %s", deployment.Spec.Template.Spec.Containers[0].Image) -// } -// if *deployment.Spec.Replicas != 5 { -// return fmt.Errorf("the replicas of nginx is not 3 but %d", *deployment.Spec.Replicas) -// } -// } -// } -// return nil -// }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(Succeed()) -// }) -// It("YurtAppOverrider should refresh template after it is updated", func() { -// By("Deployment will be returned to former when the YurtAppOverrider is deleted") -// yurtAppOverrider := &v1alpha1.YurtAppOverrider{} -// Eventually(func() error { -// return k8sClient.Get(ctx, types.NamespacedName{Name: yurtAppOverriderName, Namespace: namespaceName}, yurtAppOverrider) -// }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(Succeed()) -// for _, entry := range yurtAppOverrider.Entries { -// entry.Pools = []string{} -// } -// Expect(k8sClient.Update(ctx, yurtAppOverrider)).Should(Succeed()) -// Eventually(func() error { -// deploymentList := &v1.DeploymentList{} -// if err := k8sClient.List(ctx, deploymentList, client.MatchingLabels{ -// "apps.openyurt.io/pool-name": nodePoolName, -// }); err != nil { -// return err -// } -// for _, deployment := range deploymentList.Items { -// if deployment.Spec.Template.Spec.Containers[0].Image != "nginx:1.18" { -// return fmt.Errorf("the image of nginx is not nginx:1.18 but %s", deployment.Spec.Template.Spec.Containers[0].Image) -// } -// if *deployment.Spec.Replicas != 3 { -// return fmt.Errorf("the replicas of nginx is not 3 but %d", *deployment.Spec.Replicas) -// } -// } -// return nil -// }).WithTimeout(timeout).WithPolling(500 * time.Millisecond).Should(Succeed()) -// }) -// BeforeEach(func() { -// createNodePool() -// createYurtAppSet() -// createYurtAppOverrider() -// }) -// AfterEach(func() { -// deleteNodePool() -// deleteYurtAppSet() -// deleteYurtAppOverrider() -// }) -// }) -//}) diff --git a/test/e2e/yurt/yurtstaticset.go b/test/e2e/yurt/yurtstaticset.go index 3fe7803978c..227342ede95 100644 --- a/test/e2e/yurt/yurtstaticset.go +++ b/test/e2e/yurt/yurtstaticset.go @@ -35,7 +35,11 @@ import ( ) const ( - staticPodPath string = "/etc/kubernetes/manifests" + staticPodPath string = "/etc/kubernetes/manifests" + PodNeedUpgrade corev1.PodConditionType = "PodNeedUpgrade" + ServerName string = "127.0.0.1" + ServerPort string = "10267" + FlannelNamespace string = "kube-flannel" ) var _ = Describe("yurtStaticSet Test", Ordered, func() { @@ -53,29 +57,38 @@ var _ = Describe("yurtStaticSet Test", Ordered, func() { testImg1 := "busybox" testImg2 := "busybox:1.36.0" - createNamespace := func() { + createNamespace := func(name string) { ns := corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: namespaceName, + Name: name, }, } - Eventually( - func() error { - return k8sClient.Delete(ctx, &ns, client.PropagationPolicy(metav1.DeletePropagationForeground)) - }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) - By("make sure all the resources are removed") - - res := &corev1.Namespace{} - Eventually( - func() error { - return k8sClient.Get(ctx, client.ObjectKey{ - Name: namespaceName, - }, res) - }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(&util.NotFoundMatcher{}) - Eventually( - func() error { - return k8sClient.Create(ctx, &ns) - }).WithTimeout(timeout).WithPolling(time.Millisecond * 300).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + + // Delete the namespace if it exists. Ignore the NotFound error. + err := k8sClient.Delete(ctx, &ns, client.PropagationPolicy(metav1.DeletePropagationForeground)) + Expect(client.IgnoreNotFound(err)).To(BeNil()) + + // Wait for the namespace to be fully deleted if it was present. + if client.IgnoreNotFound(err) != nil { + // Namespace deletion was initiated, wait for completion. + Eventually( + func() bool { + return client.IgnoreNotFound(k8sClient.Get(ctx, client.ObjectKey{Name: name}, &ns)) == nil + }, + timeout, + time.Millisecond*500, + ).Should(BeTrue(), "Namespace should be deleted") + } + By("All resources in the namespace have been removed") + + // Create the namespace and expect no error or that it already exists. + err = k8sClient.Create(ctx, &ns) + Expect(client.IgnoreAlreadyExists(err)).To(BeNil()) + if client.IgnoreAlreadyExists(err) != nil { + By(fmt.Sprintf("couldn't create namespace %s", name)) + } else { + By(fmt.Sprintf("Namespace %s created", name)) + } } createStaticPod := func(nodeName string) { @@ -113,64 +126,68 @@ spec: } createYurtStaticSet := func() { - Eventually(func() error { - return k8sClient.Delete(ctx, &v1alpha1.YurtStaticSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: yurtStaticSetName, - Namespace: namespaceName, - }, - }) - }).WithTimeout(timeout).WithPolling(time.Millisecond * 300).Should(SatisfyAny(BeNil(), &util.NotFoundMatcher{})) - - testLabel := map[string]string{"app": podName} - - testYurtStaticSet := &v1alpha1.YurtStaticSet{ + yss := &v1alpha1.YurtStaticSet{ ObjectMeta: metav1.ObjectMeta{ Name: yurtStaticSetName, Namespace: namespaceName, }, - Spec: v1alpha1.YurtStaticSetSpec{ - StaticPodManifest: podName, - UpgradeStrategy: v1alpha1.YurtStaticSetUpgradeStrategy{ - Type: v1alpha1.YurtStaticSetUpgradeStrategyType(updateStrategyType), + } + + // Attempt to delete any existing YurtStaticSet with a given name and namespace. + Eventually(func() error { + return client.IgnoreNotFound(k8sClient.Delete(ctx, yss)) + }).WithTimeout(timeout).WithPolling(time.Millisecond * 300).Should(BeNil()) + + // Define the pod labels and YurtStaticSet specs. + testLabel := map[string]string{"app": podName} + yss.Spec = v1alpha1.YurtStaticSetSpec{ + StaticPodManifest: podName, + UpgradeStrategy: v1alpha1.YurtStaticSetUpgradeStrategy{ + Type: v1alpha1.YurtStaticSetUpgradeStrategyType(updateStrategyType), + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: testLabel, }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Namespace: namespaceName, - Labels: testLabel, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: testContainerName, - Image: testImg1, - Command: []string{"/bin/sh"}, - Args: []string{"-c", "while true; do echo hello; sleep 10; done"}, - }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: testContainerName, + Image: testImg1, + Command: []string{"/bin/sh"}, + Args: []string{"-c", "while true; do echo hello; sleep 10; done"}, }, }, }, }, } - + // Attempt to create a new YurtStaticSet. Eventually(func() error { - return k8sClient.Create(ctx, testYurtStaticSet) - }).WithTimeout(timeout).WithPolling(time.Millisecond * 300).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + return client.IgnoreAlreadyExists(k8sClient.Create(ctx, yss)) + }).WithTimeout(timeout).WithPolling(time.Millisecond * 300).Should(BeNil()) } updateYurtStaticSet := func() { + // Define the object key to identify the YurtStaticSet. + key := client.ObjectKey{ + Name: yurtStaticSetName, + Namespace: namespaceName, + } + + // Retries updating the YurtStaticSet until it succeeds or times out. Eventually(func() error { + // Fetch the existing YurtStaticSet instance. testYurtStaticSet := &v1alpha1.YurtStaticSet{} - if err := k8sClient.Get(ctx, client.ObjectKey{ - Name: yurtStaticSetName, - Namespace: namespaceName, - }, testYurtStaticSet); err != nil { + if err := k8sClient.Get(ctx, key, testYurtStaticSet); err != nil { return err } + + // Update the image of the first container. testYurtStaticSet.Spec.Template.Spec.Containers[0].Image = testImg2 + + // Submit the updated YurtStaticSet to the Kubernetes API server. return k8sClient.Update(ctx, testYurtStaticSet) - }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(SatisfyAny(BeNil())) + }, timeout, 500*time.Millisecond).Should(BeNil()) // Directly assert that the error should be nil. } checkPodStatusAndUpdate := func() { @@ -215,6 +232,7 @@ spec: Eventually(func() error { return checkNodeStatus(nodeName) }).WithTimeout(120 * time.Second).WithPolling(1 * time.Second).Should(Succeed()) + By("node " + nodeName + "becomes ready again") // restart flannel pod on node to recover flannel NIC Eventually(func() error { @@ -241,7 +259,7 @@ spec: nodeToImageMap = map[string]string{} k8sClient = ycfg.YurtE2eCfg.RuntimeClient namespaceName = "yurtstaticset-e2e-test" + "-" + rand.String(4) - createNamespace() + createNamespace(namespaceName) }) AfterEach(func() { @@ -250,7 +268,7 @@ spec: deleteStaticPod("openyurt-e2e-test-worker2") By(fmt.Sprintf("Delete the entire namespaceName %s", namespaceName)) - Expect(k8sClient.Delete(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespaceName}}, client.PropagationPolicy(metav1.DeletePropagationBackground))).Should(BeNil()) + Expect(k8sClient.Delete(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespaceName}}, client.PropagationPolicy(metav1.DeletePropagationForeground))).Should(BeNil()) }) Describe("Test YurtStaticSet AdvancedRollingUpdate upgrade model", func() { @@ -263,9 +281,11 @@ spec: Eventually(func() error { return checkNodeStatus("openyurt-e2e-test-worker2") }).WithTimeout(120 * time.Second).WithPolling(1 * time.Second).Should(SatisfyAll(HaveOccurred(), Not(&util.NotFoundMatcher{}))) + By("node openyurt-e2e-test-worker2 is not ready, then start to update yurtstaticset") // update the yurtStaticSet updateYurtStaticSet() + By("yurtstaticset is updated, start to check pod image") // check image version Eventually(func() error { @@ -275,9 +295,11 @@ spec: } return fmt.Errorf("error image update") }).WithTimeout(timeout * 2).WithPolling(time.Millisecond * 1000).Should(Succeed()) + By("pod on node openyurt-e2e-test-worker is updated, then start to reconnect the node openyurt-e2e-test-worker2") // recover network environment reconnectNode("openyurt-e2e-test-worker2") + By("node openyurt-e2e-test-worker2 is reconnected, and start to wait pod upgrade on node openyurt-e2e-test-worker2") // check image version Eventually(func() error { @@ -287,6 +309,7 @@ spec: } return fmt.Errorf("error image update") }).WithTimeout(timeout).WithPolling(time.Millisecond * 500).Should(Succeed()) + By("pod on node openyurt-e2e-test-worker2 is updated") }) It("Testing situation where upgrade is not required", func() {