diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e74db3a6f649..290deaeed628 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -19,18 +19,18 @@ updates: - package-ecosystem: docker directory: /cluster/images/ - target-branch: "release-1.9" + target-branch: "release-1.10" schedule: interval: weekly - package-ecosystem: docker directory: /cluster/images/ - target-branch: "release-1.8" + target-branch: "release-1.9" schedule: interval: weekly - package-ecosystem: docker directory: /cluster/images/ - target-branch: "release-1.7" + target-branch: "release-1.8" schedule: interval: weekly diff --git a/.github/workflows/ci-image-scanning.yaml b/.github/workflows/ci-image-scanning.yaml index 5b4c3cfec153..f4b431dd0d70 100644 --- a/.github/workflows/ci-image-scanning.yaml +++ b/.github/workflows/ci-image-scanning.yaml @@ -5,8 +5,12 @@ on: # for PRs initiated by Dependabot. branches-ignore: - 'dependabot/**' +permissions: + contents: read jobs: use-trivy-to-scan-image: + permissions: + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results name: image-scanning if: ${{ github.repository == 'karmada-io/karmada' }} runs-on: ubuntu-22.04 @@ -34,7 +38,7 @@ jobs: export REGISTRY="docker.io/karmada" make image-${{ matrix.target }} - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.21.0 + uses: aquasecurity/trivy-action@0.23.0 with: image-ref: 'docker.io/karmada/${{ matrix.target }}:latest' format: 'sarif' @@ -42,7 +46,7 @@ jobs: vuln-type: 'os,library' output: 'trivy-results.sarif' - name: display scan results - uses: aquasecurity/trivy-action@0.21.0 + uses: aquasecurity/trivy-action@0.23.0 with: image-ref: 'docker.io/karmada/${{ matrix.target }}:latest' format: 'table' diff --git a/.github/workflows/ci-schedule-compatibility.yaml b/.github/workflows/ci-schedule-compatibility.yaml index 8c3341da69e5..bc67d091c829 100644 --- a/.github/workflows/ci-schedule-compatibility.yaml +++ b/.github/workflows/ci-schedule-compatibility.yaml @@ -17,7 +17,9 @@ jobs: fail-fast: false matrix: kubeapiserver-version: [ v1.23.4, v1.24.2, v1.25.0, v1.26.0, v1.27.3, v1.28.0, v1.29.0, v1.30.0 ] - karmada-version: [ release-1.7, release-1.8, release-1.9 ] + karmada-version: [ master, release-1.10, release-1.9, release-1.8 ] + env: + KARMADA_APISERVER_VERSION: ${{ matrix.kubeapiserver-version }} steps: # Free up disk space on Ubuntu - name: Free Disk Space (Ubuntu) @@ -50,21 +52,6 @@ jobs: timeout_minutes: 20 command: | hack/local-up-karmada.sh - - name: change kube-apiserver and kube-controller-manager version - run: | - # Update images - kubectl --kubeconfig=${HOME}/.kube/karmada.config --context=karmada-host \ - set image deployment/karmada-apiserver -nkarmada-system \ - karmada-apiserver=registry.k8s.io/kube-apiserver:${{ matrix.kubeapiserver-version }} - kubectl --kubeconfig=${HOME}/.kube/karmada.config --context=karmada-host \ - set image deployment/karmada-kube-controller-manager -nkarmada-system \ - kube-controller-manager=registry.k8s.io/kube-controller-manager:${{ matrix.kubeapiserver-version }} - - # Wait ready - kubectl --kubeconfig=${HOME}/.kube/karmada.config --context=karmada-host \ - rollout status deployment/karmada-kube-controller-manager -nkarmada-system --timeout=5m - kubectl --kubeconfig=${HOME}/.kube/karmada.config --context=karmada-host \ - rollout status deployment/karmada-apiserver -nkarmada-system --timeout=5m - name: run e2e run: | export ARTIFACTS_PATH=${{ github.workspace }}/karmada-e2e-logs/${{ matrix.kubeapiserver-version }}-${{ matrix.karmada-version }}/ diff --git a/.github/workflows/dockerhub-latest-chart.yml b/.github/workflows/dockerhub-latest-chart.yml index 4093a4019fee..6b6a7ca7ae07 100644 --- a/.github/workflows/dockerhub-latest-chart.yml +++ b/.github/workflows/dockerhub-latest-chart.yml @@ -3,6 +3,9 @@ on: push: branches: - master + +permissions: read-all + jobs: publish-chart-to-dockerhub: name: publish to DockerHub diff --git a/.github/workflows/dockerhub-latest-image.yml b/.github/workflows/dockerhub-latest-image.yml index fe209083b9d0..76c6dcbd012c 100644 --- a/.github/workflows/dockerhub-latest-image.yml +++ b/.github/workflows/dockerhub-latest-image.yml @@ -3,6 +3,8 @@ on: push: branches: - master +permissions: + contents: read jobs: publish-image-to-dockerhub: name: publish to DockerHub diff --git a/.github/workflows/dockerhub-released-chart.yml b/.github/workflows/dockerhub-released-chart.yml index 3c1d569cf7e9..0b1d1096f29f 100644 --- a/.github/workflows/dockerhub-released-chart.yml +++ b/.github/workflows/dockerhub-released-chart.yml @@ -3,6 +3,8 @@ on: release: types: - published +permissions: + contents: read jobs: publish-chart-to-dockerhub: name: publish to DockerHub diff --git a/.github/workflows/dockerhub-released-image.yml b/.github/workflows/dockerhub-released-image.yml index a36c9cae6e87..2384499ecfd3 100644 --- a/.github/workflows/dockerhub-released-image.yml +++ b/.github/workflows/dockerhub-released-image.yml @@ -3,6 +3,8 @@ on: release: types: - published +permissions: + contents: read jobs: publish-image-to-dockerhub: name: publish to DockerHub diff --git a/.github/workflows/fossa.yml b/.github/workflows/fossa.yml index f28b396ed7e6..829ea2607e54 100644 --- a/.github/workflows/fossa.yml +++ b/.github/workflows/fossa.yml @@ -5,6 +5,10 @@ on: # for PRs initiated by Dependabot. branches-ignore: - 'dependabot/**' + +permissions: + contents: read # Required by actions/checkout@v4 to fetch the repository contents. + jobs: fossa: name: FOSSA diff --git a/.github/workflows/lint-chart.yaml b/.github/workflows/lint-chart.yaml index 23e244600804..5e9276a8ba28 100644 --- a/.github/workflows/lint-chart.yaml +++ b/.github/workflows/lint-chart.yaml @@ -17,6 +17,9 @@ on: paths: - "charts/**" +permissions: + contents: read + jobs: chart-lint-test: runs-on: ubuntu-22.04 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d6e07857f706..e5c30b86c180 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,8 +3,12 @@ on: types: - published name: Build Release +permissions: + contents: read jobs: release-assests: + permissions: + contents: write # for softprops/action-gh-release to create GitHub release name: release kubectl-karmada runs-on: ubuntu-22.04 strategy: @@ -41,6 +45,8 @@ jobs: _output/release/${{ matrix.target }}-${{ matrix.os }}-${{ matrix.arch }}.tgz _output/release/${{ matrix.target }}-${{ matrix.os }}-${{ matrix.arch }}.tgz.sha256 release-crds-assests: + permissions: + contents: write # for softprops/action-gh-release to create GitHub release name: release crds runs-on: ubuntu-22.04 steps: @@ -61,6 +67,8 @@ jobs: files: | crds.tar.gz release-charts: + permissions: + contents: write # for softprops/action-gh-release to create GitHub release name: Release charts runs-on: ubuntu-22.04 steps: @@ -78,6 +86,28 @@ jobs: _output/charts/karmada-chart-${{ github.ref_name }}.tgz.sha256 _output/charts/karmada-operator-chart-${{ github.ref_name }}.tgz _output/charts/karmada-operator-chart-${{ github.ref_name }}.tgz.sha256 + sbom-assests: + permissions: + contents: write # for softprops/action-gh-release to create GitHub release + name: Release sbom + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - name: Generate sbom for karmada file system + uses: aquasecurity/trivy-action@0.23.0 + with: + scan-type: 'fs' + format: 'spdx' + output: 'sbom-karmada.spdx' + scan-ref: "/github/workspace/" + - name: Tar the sbom files + run: | + tar -zcf sbom.tar.gz *.spdx + - name: Uploading sbom assets... + uses: softprops/action-gh-release@v2 + with: + files: | + sbom.tar.gz update-krew-index: needs: release-assests name: Update krew-index diff --git a/.github/workflows/swr-latest-image.yml b/.github/workflows/swr-latest-image.yml index f492b37c5159..aa344b53f30a 100644 --- a/.github/workflows/swr-latest-image.yml +++ b/.github/workflows/swr-latest-image.yml @@ -3,6 +3,8 @@ on: push: branches: - master +permissions: + contents: read jobs: publish-image: name: publish images diff --git a/.github/workflows/swr-released-image.yml b/.github/workflows/swr-released-image.yml index 4880ad3f6326..83730446c8ab 100644 --- a/.github/workflows/swr-released-image.yml +++ b/.github/workflows/swr-released-image.yml @@ -3,6 +3,8 @@ on: release: types: - published +permissions: + contents: read jobs: release-image: name: release images diff --git a/ADOPTERS.md b/ADOPTERS.md index 8fd435f92414..fd40f4ad3be4 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -2,6 +2,6 @@ Karmada has been adopted by many [organizations](https://karmada.io/adopters/). -If you are using Karmada in your organization, please feel free to add your name to the [list](https://karmada.io/docs/casestudies/adopters)! We are happy and proud to have you all as part of our community!πŸ’– +If you are using Karmada in your organization, please feel free to add your name to the list! We are happy and proud to have you all as part of our community!πŸ’– -To join this list, please follow [these instructions](https://karmada.io/adopters). +To join this list, please leave a comment on the [issue](https://github.com/karmada-io/karmada/issues/4540), and then the community member will help you on board. diff --git a/OWNERS b/OWNERS index 4da09d940351..60711120da26 100644 --- a/OWNERS +++ b/OWNERS @@ -4,6 +4,7 @@ reviewers: - jwcesign - Poor12 - RainbowMango +- whitewindmills - XiShanYongYe-Chang approvers: - chaunceyjiang diff --git a/README.md b/README.md index 81530c3c33ac..d4770e60914d 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,13 @@ [![Releases](https://img.shields.io/github/v/release/karmada-io/karmada)](https://github.com/karmada-io/karmada/releases/latest) [![Slack](https://img.shields.io/badge/slack-join-brightgreen)](https://slack.cncf.io) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/5301/badge)](https://bestpractices.coreinfrastructure.org/projects/5301) +[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/karmada-io/karmada/badge)](https://securityscorecards.dev/viewer/?uri=github.com/karmada-io/karmada) ![build](https://github.com/karmada-io/karmada/actions/workflows/ci.yml/badge.svg) [![Go Report Card](https://goreportcard.com/badge/github.com/karmada-io/karmada)](https://goreportcard.com/report/github.com/karmada-io/karmada) [![codecov](https://codecov.io/gh/karmada-io/karmada/branch/master/graph/badge.svg?token=ROM8CMPXZ6)](https://codecov.io/gh/karmada-io/karmada) +[![FOSSA Status](https://app.fossa.com/api/projects/custom%2B28176%2Fgithub.com%2Fkarmada-io%2Fkarmada.svg?type=shield)](https://app.fossa.com/projects/custom%2B28176%2Fgithub.com%2Fkarmada-io%2Fkarmada?ref=badge_shield) +[![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/karmada)](https://artifacthub.io/packages/krew/krew-index/karmada) +[![CLOMonitor](https://img.shields.io/endpoint?url=https://clomonitor.io/api/projects/cncf/karmada/badge)](https://clomonitor.io/projects/cncf/karmada) ## Karmada: Open, Multi-Cloud, Multi-Cluster Kubernetes Orchestration diff --git a/artifacts/deploy/karmada-apiserver.yaml b/artifacts/deploy/karmada-apiserver.yaml index 4cdf040afeb8..296f7b8f0a9a 100644 --- a/artifacts/deploy/karmada-apiserver.yaml +++ b/artifacts/deploy/karmada-apiserver.yaml @@ -64,7 +64,7 @@ spec: - --tls-private-key-file=/etc/karmada/pki/apiserver.key - --tls-min-version=VersionTLS13 name: karmada-apiserver - image: registry.k8s.io/kube-apiserver:v1.27.11 + image: registry.k8s.io/kube-apiserver:{{karmada_apiserver_version}} imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 8 diff --git a/artifacts/deploy/karmada-descheduler.yaml b/artifacts/deploy/karmada-descheduler.yaml index c451a37b4312..53d0d189bd7c 100644 --- a/artifacts/deploy/karmada-descheduler.yaml +++ b/artifacts/deploy/karmada-descheduler.yaml @@ -27,6 +27,9 @@ spec: - /bin/karmada-descheduler - --kubeconfig=/etc/kubeconfig - --bind-address=0.0.0.0 + - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt + - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt + - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key - --v=4 livenessProbe: httpGet: @@ -38,10 +41,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: karmada-certs + mountPath: /etc/karmada/pki + readOnly: true - name: kubeconfig subPath: kubeconfig mountPath: /etc/kubeconfig volumes: + - name: karmada-certs + secret: + secretName: karmada-cert-secret - name: kubeconfig secret: secretName: kubeconfig diff --git a/artifacts/deploy/karmada-scheduler-estimator.yaml b/artifacts/deploy/karmada-scheduler-estimator.yaml index ce4af9361e1a..85d6111f5aae 100644 --- a/artifacts/deploy/karmada-scheduler-estimator.yaml +++ b/artifacts/deploy/karmada-scheduler-estimator.yaml @@ -27,6 +27,9 @@ spec: - /bin/karmada-scheduler-estimator - --kubeconfig=/etc/{{member_cluster_name}}-kubeconfig - --cluster-name={{member_cluster_name}} + - --grpc-auth-cert-file=/etc/karmada/pki/karmada.crt + - --grpc-auth-key-file=/etc/karmada/pki/karmada.key + - --grpc-client-ca-file=/etc/karmada/pki/ca.crt livenessProbe: httpGet: path: /healthz @@ -37,10 +40,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: karmada-certs + mountPath: /etc/karmada/pki + readOnly: true - name: member-kubeconfig subPath: {{member_cluster_name}}-kubeconfig mountPath: /etc/{{member_cluster_name}}-kubeconfig volumes: + - name: karmada-certs + secret: + secretName: karmada-cert-secret - name: member-kubeconfig secret: secretName: {{member_cluster_name}}-kubeconfig diff --git a/artifacts/deploy/karmada-scheduler.yaml b/artifacts/deploy/karmada-scheduler.yaml index 1401e7a4cca0..f863fba55a3a 100644 --- a/artifacts/deploy/karmada-scheduler.yaml +++ b/artifacts/deploy/karmada-scheduler.yaml @@ -38,12 +38,21 @@ spec: - --bind-address=0.0.0.0 - --secure-port=10351 - --enable-scheduler-estimator=true + - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt + - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt + - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key - --v=4 volumeMounts: + - name: karmada-certs + mountPath: /etc/karmada/pki + readOnly: true - name: kubeconfig subPath: kubeconfig mountPath: /etc/kubeconfig volumes: + - name: karmada-certs + secret: + secretName: karmada-cert-secret - name: kubeconfig secret: secretName: kubeconfig diff --git a/artifacts/deploy/kube-controller-manager.yaml b/artifacts/deploy/kube-controller-manager.yaml index b74f2a90c687..205759193f3c 100644 --- a/artifacts/deploy/kube-controller-manager.yaml +++ b/artifacts/deploy/kube-controller-manager.yaml @@ -52,7 +52,7 @@ spec: - --service-cluster-ip-range=10.96.0.0/12 - --use-service-account-credentials=true - --v=4 - image: registry.k8s.io/kube-controller-manager:v1.27.11 + image: registry.k8s.io/kube-controller-manager:{{karmada_apiserver_version}} imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 8 diff --git a/artifacts/kindClusterConfig/member3.yaml b/artifacts/kindClusterConfig/member3.yaml new file mode 100644 index 000000000000..f8faaebad6af --- /dev/null +++ b/artifacts/kindClusterConfig/member3.yaml @@ -0,0 +1,7 @@ +kind: Cluster +apiVersion: "kind.x-k8s.io/v1alpha4" +networking: + podSubnet: "10.14.0.0/16" + serviceSubnet: "10.15.0.0/16" +nodes: + - role: control-plane diff --git a/charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml b/charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml index f919ca2b94d1..bed6b2796fb7 100644 --- a/charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml +++ b/charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml @@ -18,10 +18,10 @@ spec: versions: - additionalPrinterColumns: - jsonPath: .status.conditions[?(@.type=="Ready")].status - name: Ready + name: READY type: string - jsonPath: .metadata.creationTimestamp - name: Age + name: AGE type: date name: v1alpha1 schema: diff --git a/charts/karmada/README.md b/charts/karmada/README.md index aa8f642a1d5c..e3ff8fbe169c 100644 --- a/charts/karmada/README.md +++ b/charts/karmada/README.md @@ -308,7 +308,7 @@ helm install karmada-scheduler-estimator -n karmada-system ./charts/karmada | `apiServer.podAnnotations` | Annotations of the karmada-apiserver pods | `{}` | | `apiServer.image.pullSecrets` | Image pull secret of the karmada-apiserver | `[]` | | `apiServer.image.repository` | Image of the karmada-apiserver | `"registry.k8s.io/kube-apiserver"` | -| `apiServer.image.tag` | Image tag of the karmada-apiserver | `"v1.27.11"` | +| `apiServer.image.tag` | Image tag of the karmada-apiserver | `"v1.28.9"` | | `apiServer.image.pullPolicy` | Image pull policy of the karmada-apiserver | `"IfNotPresent"` | | `apiServer.resources` | Resource quota of the karmada-apiserver | `{}` | | `apiServer.hostNetwork` | Deploy karmada-apiserver with hostNetwork. If there are multiple karmadas in one cluster, you'd better set it to "false" | `"false"` | @@ -337,7 +337,7 @@ helm install karmada-scheduler-estimator -n karmada-system ./charts/karmada | `kubeControllerManager.podAnnotations` | Annotations of the kube-controller-manager pods | `{}` | | `kubeControllerManager.image.pullSecrets` | Image pull secret of the kube-controller-manager | `[]` | | `kubeControllerManager.image.repository` | Image of the kube-controller-manager | `"registry.k8s.io/kube-controller-manager"` | -| `kubeControllerManager.image.tag` | Image tag of the kube-controller-manager | `"v1.27.11"` | +| `kubeControllerManager.image.tag` | Image tag of the kube-controller-manager | `"v1.28.9"` | | `kubeControllerManager.image.pullPolicy` | Image pull policy of the kube-controller-manager | `"IfNotPresent"` | | `kubeControllerManager.resources` | Resource quota of the kube-controller-manager | `{}` | | `kubeControllerManager.nodeSelector` | Node selector of the kube-controller-manager | `{}` | diff --git a/charts/karmada/_crds/bases/config/config.karmada.io_resourceinterpretercustomizations.yaml b/charts/karmada/_crds/bases/config/config.karmada.io_resourceinterpretercustomizations.yaml index 212f08552c23..394496651f12 100644 --- a/charts/karmada/_crds/bases/config/config.karmada.io_resourceinterpretercustomizations.yaml +++ b/charts/karmada/_crds/bases/config/config.karmada.io_resourceinterpretercustomizations.yaml @@ -18,7 +18,17 @@ spec: singular: resourceinterpretercustomization scope: Cluster versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .spec.target.apiVersion + name: TARGET-API-VERSION + type: string + - jsonPath: .spec.target.kind + name: TARGET-KIND + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 schema: openAPIV3Schema: description: |- @@ -372,3 +382,4 @@ spec: type: object served: true storage: true + subresources: {} diff --git a/charts/karmada/_crds/bases/policy/policy.karmada.io_clusterpropagationpolicies.yaml b/charts/karmada/_crds/bases/policy/policy.karmada.io_clusterpropagationpolicies.yaml index 7c3de0fa7a51..515ed7cf02d5 100644 --- a/charts/karmada/_crds/bases/policy/policy.karmada.io_clusterpropagationpolicies.yaml +++ b/charts/karmada/_crds/bases/policy/policy.karmada.io_clusterpropagationpolicies.yaml @@ -18,7 +18,17 @@ spec: singular: clusterpropagationpolicy scope: Cluster versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .spec.conflictResolution + name: CONFLICT-RESOLUTION + type: string + - jsonPath: .spec.priority + name: PRIORITY + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 schema: openAPIV3Schema: description: |- @@ -793,3 +803,4 @@ spec: type: object served: true storage: true + subresources: {} diff --git a/charts/karmada/_crds/bases/policy/policy.karmada.io_propagationpolicies.yaml b/charts/karmada/_crds/bases/policy/policy.karmada.io_propagationpolicies.yaml index 7866c7a48122..939a24b91fee 100644 --- a/charts/karmada/_crds/bases/policy/policy.karmada.io_propagationpolicies.yaml +++ b/charts/karmada/_crds/bases/policy/policy.karmada.io_propagationpolicies.yaml @@ -18,7 +18,17 @@ spec: singular: propagationpolicy scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .spec.conflictResolution + name: CONFLICT-RESOLUTION + type: string + - jsonPath: .spec.priority + name: PRIORITY + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 schema: openAPIV3Schema: description: PropagationPolicy represents the policy that propagates a group @@ -790,3 +800,4 @@ spec: type: object served: true storage: true + subresources: {} diff --git a/charts/karmada/_crds/bases/work/work.karmada.io_clusterresourcebindings.yaml b/charts/karmada/_crds/bases/work/work.karmada.io_clusterresourcebindings.yaml index 082d1fb03b8f..8818778736c3 100644 --- a/charts/karmada/_crds/bases/work/work.karmada.io_clusterresourcebindings.yaml +++ b/charts/karmada/_crds/bases/work/work.karmada.io_clusterresourcebindings.yaml @@ -222,13 +222,13 @@ spec: status: {} - additionalPrinterColumns: - jsonPath: .status.conditions[?(@.type=="Scheduled")].status - name: Scheduled + name: SCHEDULED type: string - jsonPath: .status.conditions[?(@.type=="FullyApplied")].status - name: FullyApplied + name: FULLYAPPLIED type: string - jsonPath: .metadata.creationTimestamp - name: Age + name: AGE type: date name: v1alpha2 schema: diff --git a/charts/karmada/_crds/bases/work/work.karmada.io_resourcebindings.yaml b/charts/karmada/_crds/bases/work/work.karmada.io_resourcebindings.yaml index 410c559adb0d..9b516147c2cc 100644 --- a/charts/karmada/_crds/bases/work/work.karmada.io_resourcebindings.yaml +++ b/charts/karmada/_crds/bases/work/work.karmada.io_resourcebindings.yaml @@ -222,13 +222,13 @@ spec: status: {} - additionalPrinterColumns: - jsonPath: .status.conditions[?(@.type=="Scheduled")].status - name: Scheduled + name: SCHEDULED type: string - jsonPath: .status.conditions[?(@.type=="FullyApplied")].status - name: FullyApplied + name: FULLYAPPLIED type: string - jsonPath: .metadata.creationTimestamp - name: Age + name: AGE type: date name: v1alpha2 schema: diff --git a/charts/karmada/_crds/bases/work/work.karmada.io_works.yaml b/charts/karmada/_crds/bases/work/work.karmada.io_works.yaml index 937fcc53aa9e..7109cd69a52b 100644 --- a/charts/karmada/_crds/bases/work/work.karmada.io_works.yaml +++ b/charts/karmada/_crds/bases/work/work.karmada.io_works.yaml @@ -20,13 +20,13 @@ spec: versions: - additionalPrinterColumns: - jsonPath: .spec.workload.manifests[*].kind - name: Workload-Kind + name: WORKLOAD-KIND type: string - jsonPath: .status.conditions[?(@.type=="Applied")].status - name: Applied + name: APPLIED type: string - jsonPath: .metadata.creationTimestamp - name: Age + name: AGE type: date name: v1alpha1 schema: diff --git a/charts/karmada/templates/_helpers.tpl b/charts/karmada/templates/_helpers.tpl index 78abfd86921d..2d129b7cea09 100644 --- a/charts/karmada/templates/_helpers.tpl +++ b/charts/karmada/templates/_helpers.tpl @@ -306,6 +306,10 @@ app: {{- include "karmada.name" .}}-search {{- include "karmada.commonLabels" . -}} {{- end -}} +{{- define "karmada.staticResourceJob.labels" -}} +{{- include "karmada.commonLabels" . -}} +{{- end -}} + {{- define "karmada.postInstallJob.labels" -}} {{- include "karmada.commonLabels" . -}} {{- end -}} @@ -345,6 +349,13 @@ app: {{- include "karmada.name" .}}-search {{- end }} {{- end -}} +{{- define "karmada.scheduler.cert.volume" -}} +{{ $name := include "karmada.name" . }} +- name: karmada-certs + secret: + secretName: {{ $name }}-cert +{{- end -}} + {{/* Return the proper karmada internal etcd image name */}} @@ -574,3 +585,55 @@ Return the proper Docker Image Registry Secret Names {{- end }} {{- end }} {{- end -}} + +{{- define "karmada.init-sa-secret.volume" -}} +{{- $name := include "karmada.name" . -}} +- name: init-sa-secret + secret: + secretName: {{ $name }}-hook-job +{{- end -}} + +{{- define "karmada.init-sa-secret.volumeMount" -}} +- name: init-sa-secret + mountPath: /opt/mount +{{- end -}} + +{{- define "karmada.initContainer.build-kubeconfig" -}} +TOKEN=$(cat /opt/mount/token) +kubectl config set-cluster karmada-host --server=https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT} --certificate-authority=/opt/mount/ca.crt +kubectl config set-credentials default --token=$TOKEN +kubectl config set-context karmada-host-context --cluster=karmada-host --user=default --namespace=default +kubectl config use-context karmada-host-context +{{- end -}} + +{{- define "karmada.initContainer.waitEtcd" -}} +- name: wait + image: {{ include "karmada.kubectl.image" . }} + imagePullPolicy: {{ .Values.kubectl.image.pullPolicy }} + command: + - /bin/sh + - -c + - | + bash <<'EOF' + {{- include "karmada.initContainer.build-kubeconfig" . | nindent 6 }} + kubectl rollout status statefulset etcd -n {{ include "karmada.namespace" . }} + EOF + volumeMounts: + {{- include "karmada.init-sa-secret.volumeMount" .| nindent 4 }} +{{- end -}} + +{{- define "karmada.initContainer.waitStaticResource" -}} +- name: wait + image: {{ include "karmada.kubectl.image" . }} + imagePullPolicy: {{ .Values.kubectl.image.pullPolicy }} + command: + - /bin/sh + - -c + - | + bash <<'EOF' + {{- include "karmada.initContainer.build-kubeconfig" . | nindent 6 }} + kubectl wait --for=condition=complete job {{ include "karmada.name" . }}-static-resource -n {{ include "karmada.namespace" . }} + EOF + volumeMounts: + {{- include "karmada.init-sa-secret.volumeMount" .| nindent 4 }} +{{- end -}} diff --git a/charts/karmada/templates/karmada-aggregated-apiserver.yaml b/charts/karmada/templates/karmada-aggregated-apiserver.yaml index 3d572c5f4f74..cc5e45fdaa35 100644 --- a/charts/karmada/templates/karmada-aggregated-apiserver.yaml +++ b/charts/karmada/templates/karmada-aggregated-apiserver.yaml @@ -29,6 +29,8 @@ spec: spec: {{- include "karmada.aggregatedApiServer.imagePullSecrets" . | nindent 6 }} automountServiceAccountToken: false + initContainers: + {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }} containers: - name: {{ $name }}-aggregated-apiserver image: {{ template "karmada.aggregatedApiServer.image" . }} @@ -96,6 +98,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: + {{- include "karmada.init-sa-secret.volume" . | nindent 8 }} {{- include "karmada.kubeconfig.volume" . | nindent 8 }} - name: apiserver-cert secret: diff --git a/charts/karmada/templates/karmada-apiserver.yaml b/charts/karmada/templates/karmada-apiserver.yaml index a788d2d2be2b..4c561ec240fd 100644 --- a/charts/karmada/templates/karmada-apiserver.yaml +++ b/charts/karmada/templates/karmada-apiserver.yaml @@ -28,6 +28,8 @@ spec: spec: {{- include "karmada.apiServer.imagePullSecrets" . | nindent 6 }} automountServiceAccountToken: false + initContainers: + {{- include "karmada.initContainer.waitEtcd" . | nindent 8 }} containers: - name: {{ $name }}-apiserver image: {{ template "karmada.apiServer.image" . }} @@ -135,6 +137,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: + {{- include "karmada.init-sa-secret.volume" . | nindent 8 }} - name: apiserver-cert secret: secretName: {{ $name }}-cert diff --git a/charts/karmada/templates/karmada-controller-manager.yaml b/charts/karmada/templates/karmada-controller-manager.yaml index 2a4e565f3d49..3415573ac66b 100644 --- a/charts/karmada/templates/karmada-controller-manager.yaml +++ b/charts/karmada/templates/karmada-controller-manager.yaml @@ -42,7 +42,10 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: + {{- include "karmada.init-sa-secret.volume" . | nindent 8 }} {{- include "karmada.kubeconfig.volume" . | nindent 8 }} + initContainers: + {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }} containers: - name: {{ $name }}-controller-manager image: {{ template "karmada.controllerManager.image" . }} diff --git a/charts/karmada/templates/karmada-descheduler.yaml b/charts/karmada/templates/karmada-descheduler.yaml index caf9b8d29757..a74a7cd247fa 100644 --- a/charts/karmada/templates/karmada-descheduler.yaml +++ b/charts/karmada/templates/karmada-descheduler.yaml @@ -41,6 +41,8 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} automountServiceAccountToken: false + initContainers: + {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }} containers: - name: {{ $name }}-descheduler image: {{ template "karmada.descheduler.image" . }} @@ -50,6 +52,9 @@ spec: - --kubeconfig=/etc/kubeconfig - --bind-address=0.0.0.0 - --leader-elect-resource-namespace={{ $systemNamespace }} + - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt + - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt + - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key - --v=4 livenessProbe: httpGet: @@ -61,11 +66,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: karmada-certs + mountPath: /etc/karmada/pki + readOnly: true {{- include "karmada.kubeconfig.volumeMount" . | nindent 12 }} resources: {{- toYaml .Values.descheduler.resources | nindent 12 }} volumes: + {{- include "karmada.init-sa-secret.volume" . | nindent 8 }} {{- include "karmada.descheduler.kubeconfig.volume" . | nindent 8 }} + {{- include "karmada.scheduler.cert.volume" . | nindent 8 }} {{ if .Values.descheduler.podDisruptionBudget }} --- diff --git a/charts/karmada/templates/karmada-metrics-adapter.yaml b/charts/karmada/templates/karmada-metrics-adapter.yaml index 49f8fe29e0ec..16bc12bb5dd7 100644 --- a/charts/karmada/templates/karmada-metrics-adapter.yaml +++ b/charts/karmada/templates/karmada-metrics-adapter.yaml @@ -30,6 +30,8 @@ spec: spec: {{- include "karmada.metricsAdapter.imagePullSecrets" . | nindent 6 }} automountServiceAccountToken: false + initContainers: + {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }} containers: - name: {{ $name }}-aggregated-apiserver image: {{ template "karmada.metricsAdapter.image" . }} @@ -81,6 +83,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: + {{- include "karmada.init-sa-secret.volume" . | nindent 8 }} {{- include "karmada.kubeconfig.volume" . | nindent 8 }} - name: apiserver-cert secret: diff --git a/charts/karmada/templates/karmada-scheduler-estimator.yaml b/charts/karmada/templates/karmada-scheduler-estimator.yaml index 33bdd7846911..55d6e548eb6c 100644 --- a/charts/karmada/templates/karmada-scheduler-estimator.yaml +++ b/charts/karmada/templates/karmada-scheduler-estimator.yaml @@ -48,6 +48,9 @@ spec: - /bin/karmada-scheduler-estimator - --kubeconfig=/etc/{{ $clusterName }}-kubeconfig - --cluster-name={{ $clusterName }} + - --grpc-auth-cert-file=/etc/karmada/pki/karmada.crt + - --grpc-auth-key-file=/etc/karmada/pki/karmada.key + - --grpc-client-ca-file=/etc/karmada/pki/ca.crt {{- with (include "karmada.schedulerEstimator.featureGates" (dict "featureGatesArg" $.Values.schedulerEstimator.featureGates)) }} - {{ . }} {{- end}} @@ -61,12 +64,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: karmada-certs + mountPath: /etc/karmada/pki + readOnly: true - name: member-kubeconfig subPath: {{ $clusterName }}-kubeconfig mountPath: /etc/{{ $clusterName }}-kubeconfig resources: {{- toYaml $.Values.schedulerEstimator.resources | nindent 12 }} volumes: + {{- include "karmada.scheduler.cert.volume" $ | nindent 8 }} - name: member-kubeconfig secret: secretName: {{ $clusterName }}-kubeconfig diff --git a/charts/karmada/templates/karmada-scheduler.yaml b/charts/karmada/templates/karmada-scheduler.yaml index 503c050b16b2..9ef8b60c4656 100644 --- a/charts/karmada/templates/karmada-scheduler.yaml +++ b/charts/karmada/templates/karmada-scheduler.yaml @@ -41,6 +41,8 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} automountServiceAccountToken: false + initContainers: + {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }} containers: - name: {{ $name }}-scheduler image: {{ template "karmada.scheduler.image" .}} @@ -51,6 +53,9 @@ spec: - --bind-address=0.0.0.0 - --secure-port=10351 - --leader-elect-resource-namespace={{ $systemNamespace }} + - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt + - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt + - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key livenessProbe: httpGet: path: /healthz @@ -61,11 +66,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: karmada-certs + mountPath: /etc/karmada/pki + readOnly: true {{- include "karmada.kubeconfig.volumeMount" . | nindent 12 }} resources: {{- toYaml .Values.scheduler.resources | nindent 12 }} volumes: + {{- include "karmada.init-sa-secret.volume" . | nindent 8 }} {{- include "karmada.kubeconfig.volume" . | nindent 8 }} + {{- include "karmada.scheduler.cert.volume" . | nindent 8 }} {{ if .Values.scheduler.podDisruptionBudget }} --- diff --git a/charts/karmada/templates/karmada-search.yaml b/charts/karmada/templates/karmada-search.yaml index 5478b46cb486..bb4c91fa9824 100644 --- a/charts/karmada/templates/karmada-search.yaml +++ b/charts/karmada/templates/karmada-search.yaml @@ -40,6 +40,8 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} automountServiceAccountToken: false + initContainers: + {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }} containers: - name: {{ $name }}-search image: {{ template "karmada.search.image" . }} @@ -90,6 +92,7 @@ spec: resources: {{- toYaml .Values.apiServer.resources | nindent 12 }} volumes: + {{- include "karmada.init-sa-secret.volume" . | nindent 8 }} {{- include "karmada.search.kubeconfig.volume" . | nindent 8 }} {{- include "karmada.search.etcd.cert.volume" . | nindent 8 }} --- diff --git a/charts/karmada/templates/karmada-static-resource-job.yaml b/charts/karmada/templates/karmada-static-resource-job.yaml new file mode 100644 index 000000000000..1d2ea6179910 --- /dev/null +++ b/charts/karmada/templates/karmada-static-resource-job.yaml @@ -0,0 +1,105 @@ +{{- $name := include "karmada.name" . -}} +{{- $namespace := include "karmada.namespace" . -}} +{{- if eq .Values.installMode "host" }} +apiVersion: batch/v1 +kind: Job +metadata: + name: "{{ $name }}-static-resource" + namespace: {{ $namespace }} + labels: + {{- include "karmada.staticResourceJob.labels" . | nindent 4 }} +spec: + parallelism: 1 + completions: 1 + template: + metadata: + name: {{ $name }} + labels: + {{- include "karmada.staticResourceJob.labels" . | nindent 8 }} + spec: + {{- include "karmada.imagePullSecrets" . | nindent 6 }} + {{- with .Values.staticResourceJob.tolerations}} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.staticResourceJob.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ $name }}-hook-job + restartPolicy: Never + containers: + - name: post-install + image: {{ template "karmada.kubectl.image" . }} + imagePullPolicy: {{ .Values.kubectl.image.pullPolicy }} + command: + - /bin/sh + - -c + - | + bash <<'EOF' + set -ex + kubectl rollout status deployment {{ $name }}-apiserver -n {{ $namespace }} + kubectl apply -k /crds --kubeconfig /etc/kubeconfig + kubectl apply -f /static-resources/system-namespace.yaml --kubeconfig /etc/kubeconfig + kubectl apply -f /static-resources/ --kubeconfig /etc/kubeconfig + EOF + volumeMounts: + - name: {{ $name }}-crds-kustomization + mountPath: /crds + - name: {{ $name }}-crds-patches + mountPath: /crds/patches + - name: {{ $name }}-crds-autoscaling-bases + mountPath: /crds/bases/autoscaling + - name: {{ $name }}-crds-config-bases + mountPath: /crds/bases/config + - name: {{ $name }}-crds-multicluster-bases + mountPath: /crds/bases/multicluster + - name: {{ $name }}-crds-networking-bases + mountPath: /crds/bases/networking + - name: {{ $name }}-crds-policy-bases + mountPath: /crds/bases/policy + - name: {{ $name }}-crds-remedy-bases + mountPath: /crds/bases/remedy + - name: {{ $name }}-crds-work-bases + mountPath: /crds/bases/work + - name: {{ $name }}-crds-apps-bases + mountPath: /crds/bases/apps + - name: {{ $name }}-static-resources + mountPath: /static-resources + {{ include "karmada.kubeconfig.volumeMount" . | nindent 10 }} + volumes: + - name: {{ $name }}-crds-kustomization + configMap: + name: {{ $name }}-crds-kustomization + - name: {{ $name }}-crds-patches + configMap: + name: {{ $name }}-crds-patches + - name: {{ $name }}-crds-autoscaling-bases + configMap: + name: {{ $name }}-crds-autoscaling-bases + - name: {{ $name }}-crds-config-bases + configMap: + name: {{ $name }}-crds-config-bases + - name: {{ $name }}-crds-multicluster-bases + configMap: + name: {{ $name }}-crds-multicluster-bases + - name: {{ $name }}-crds-networking-bases + configMap: + name: {{ $name }}-crds-networking-bases + - name: {{ $name }}-crds-policy-bases + configMap: + name: {{ $name }}-crds-policy-bases + - name: {{ $name }}-crds-remedy-bases + configMap: + name: {{ $name }}-crds-remedy-bases + - name: {{ $name }}-crds-work-bases + configMap: + name: {{ $name }}-crds-work-bases + - name: {{ $name }}-crds-apps-bases + configMap: + name: {{ $name }}-crds-apps-bases + - name: {{ $name }}-static-resources + configMap: + name: {{ $name }}-static-resources + {{ include "karmada.kubeconfig.volume" . | nindent 8 }} +{{- end }} diff --git a/charts/karmada/templates/karmada-webhook.yaml b/charts/karmada/templates/karmada-webhook.yaml index 60bd70391425..3f20dfe71406 100644 --- a/charts/karmada/templates/karmada-webhook.yaml +++ b/charts/karmada/templates/karmada-webhook.yaml @@ -41,6 +41,8 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + initContainers: + {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }} containers: - name: {{ $name }}-webhook image: {{ template "karmada.webhook.image" . }} @@ -66,6 +68,7 @@ spec: resources: {{- toYaml .Values.webhook.resources | nindent 12 }} volumes: + {{- include "karmada.init-sa-secret.volume" . | nindent 8 }} {{- include "karmada.kubeconfig.volume" . | nindent 8 }} - name: {{ $name }}-webhook-cert-secret secret: diff --git a/charts/karmada/templates/kube-controller-manager.yaml b/charts/karmada/templates/kube-controller-manager.yaml index 2e966253d18b..ab9f40223c7f 100644 --- a/charts/karmada/templates/kube-controller-manager.yaml +++ b/charts/karmada/templates/kube-controller-manager.yaml @@ -41,6 +41,8 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + initContainers: + {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }} containers: - command: - kube-controller-manager @@ -87,6 +89,7 @@ spec: - name: apisever-cert secret: secretName: {{ $name }}-cert + {{- include "karmada.init-sa-secret.volume" . | nindent 8 }} {{- include "karmada.kubeconfig.volume" . | nindent 8 }} {{ if .Values.kubeControllerManager.podDisruptionBudget }} diff --git a/charts/karmada/templates/post-delete-job.yaml b/charts/karmada/templates/post-delete-job.yaml index 699c38ac66b2..0b9907873daf 100644 --- a/charts/karmada/templates/post-delete-job.yaml +++ b/charts/karmada/templates/post-delete-job.yaml @@ -33,7 +33,7 @@ spec: nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ $name }}-pre-job + serviceAccountName: {{ $name }}-hook-job restartPolicy: Never containers: - name: post-delete @@ -47,14 +47,13 @@ spec: set -ex kubectl delete -f /opt/mount/ --ignore-not-found=true kubectl delete -f /opt/crds/ --ignore-not-found=true -R + kubectl delete -f /opt/static-resources/ --ignore-not-found=true -R kubectl delete cm/{{ $name }}-config -n {{ $namespace }} --ignore-not-found=true kubectl delete deployment/{{ $name }}-controller-manager -n {{ $namespace }} --ignore-not-found=true EOF volumeMounts: - name: mount mountPath: /opt/mount - - name: crds - mountPath: /opt/crds - name: crds-autoscaling-base mountPath: /opt/crds/base/autoscaling - name: crds-config-base @@ -69,35 +68,37 @@ spec: mountPath: /opt/crds/base/remedy - name: crds-work-base mountPath: /opt/crds/base/work + - name: static-resources + mountPath: /opt/static-resources volumes: - name: mount configMap: name: {{ $name }}-config - - name: crds - configMap: - name: {{ $name }}-crds-config - name: crds-autoscaling-base configMap: - name: {{ $name }}-crds-autoscaling-bases-config + name: {{ $name }}-crds-autoscaling-bases - name: crds-config-base configMap: - name: {{ $name }}-crds-config-bases-config + name: {{ $name }}-crds-config-bases - name: crds-multicluster-base configMap: - name: {{ $name }}-crds-multicluster-bases-config + name: {{ $name }}-crds-multicluster-bases - name: crds-networking-base configMap: - name: {{ $name }}-crds-networking-bases-config + name: {{ $name }}-crds-networking-bases - name: crds-policy-base configMap: - name: {{ $name }}-crds-policy-bases-config + name: {{ $name }}-crds-policy-bases - name: crds-remedy-base configMap: - name: {{ $name }}-crds-remedy-bases-config + name: {{ $name }}-crds-remedy-bases - name: crds-work-base configMap: - name: {{ $name }}-crds-work-bases-config + name: {{ $name }}-crds-work-bases - name: crds-apps-base configMap: - name: {{ $name }}-crds-apps-bases-config + name: {{ $name }}-crds-apps-bases + - name: static-resources + configMap: + name: {{ $name }}-static-resources {{- end }} diff --git a/charts/karmada/templates/post-install-job.yaml b/charts/karmada/templates/post-install-job.yaml index c14a43079fa7..74f3046aa909 100644 --- a/charts/karmada/templates/post-install-job.yaml +++ b/charts/karmada/templates/post-install-job.yaml @@ -38,6 +38,7 @@ spec: nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} + serviceAccountName: {{ $name }}-hook-job restartPolicy: Never containers: - name: post-install @@ -49,66 +50,14 @@ spec: - | bash <<'EOF' set -ex - kubectl apply -k /crds --kubeconfig /etc/kubeconfig - kubectl apply -f /static-resources --kubeconfig /etc/kubeconfig + + # The `post-install hook job` may be applied before all karmada components are ready that rely on `static-resource job` and `hook-job secret`. + # So, we have to postpone the deletion of the `static-resource job` and `hook-job secret` until all karmada components are up and running. + while [[ $(kubectl get pods -n {{ $namespace }} --field-selector=status.phase!=Running,status.phase!=Succeeded -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | wc -l) > 0 ]]; do + echo "waiting for all pods of karmada control plane ready..."; sleep 1; + done + + kubectl delete job {{ $name }}-static-resource -n {{ $namespace }} + kubectl delete secret {{ $name }}-hook-job -n {{ $namespace }} EOF - volumeMounts: - - name: {{ $name }}-crds-kustomization - mountPath: /crds - - name: {{ $name }}-crds-patches - mountPath: /crds/patches - - name: {{ $name }}-crds-autoscaling-bases - mountPath: /crds/bases/autoscaling - - name: {{ $name }}-crds-config-bases - mountPath: /crds/bases/config - - name: {{ $name }}-crds-multicluster-bases - mountPath: /crds/bases/multicluster - - name: {{ $name }}-crds-networking-bases - mountPath: /crds/bases/networking - - name: {{ $name }}-crds-policy-bases - mountPath: /crds/bases/policy - - name: {{ $name }}-crds-remedy-bases - mountPath: /crds/bases/remedy - - name: {{ $name }}-crds-work-bases - mountPath: /crds/bases/work - - name: {{ $name }}-crds-apps-bases - mountPath: /crds/bases/apps - - name: {{ $name }}-static-resources - mountPath: /static-resources - {{ include "karmada.kubeconfig.volumeMount" . | nindent 10 }} - volumes: - - name: {{ $name }}-crds-kustomization - configMap: - name: {{ $name }}-crds-kustomization - - name: {{ $name }}-crds-patches - configMap: - name: {{ $name }}-crds-patches - - name: {{ $name }}-crds-autoscaling-bases - configMap: - name: {{ $name }}-crds-autoscaling-bases - - name: {{ $name }}-crds-config-bases - configMap: - name: {{ $name }}-crds-config-bases - - name: {{ $name }}-crds-multicluster-bases - configMap: - name: {{ $name }}-crds-multicluster-bases - - name: {{ $name }}-crds-networking-bases - configMap: - name: {{ $name }}-crds-networking-bases - - name: {{ $name }}-crds-policy-bases - configMap: - name: {{ $name }}-crds-policy-bases - - name: {{ $name }}-crds-remedy-bases - configMap: - name: {{ $name }}-crds-remedy-bases - - name: {{ $name }}-crds-work-bases - configMap: - name: {{ $name }}-crds-work-bases - - name: {{ $name }}-crds-apps-bases - configMap: - name: {{ $name }}-crds-apps-bases - - name: {{ $name }}-static-resources - configMap: - name: {{ $name }}-static-resources - {{ include "karmada.kubeconfig.volume" . | nindent 8 }} {{- end }} diff --git a/charts/karmada/templates/pre-install-job.yaml b/charts/karmada/templates/pre-install-job.yaml index c5eca2551cce..cc8fe785f67a 100644 --- a/charts/karmada/templates/pre-install-job.yaml +++ b/charts/karmada/templates/pre-install-job.yaml @@ -372,7 +372,7 @@ spec: nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ $name }}-pre-job + serviceAccountName: {{ $name }}-hook-job restartPolicy: Never initContainers: - name: init @@ -445,12 +445,11 @@ spec: name: {{ $name }}-config - name: configs emptyDir: {} - --- apiVersion: v1 kind: ServiceAccount metadata: - name: {{ $name }}-pre-job + name: {{ $name }}-hook-job namespace: {{ $namespace }} annotations: "helm.sh/hook": pre-install @@ -460,10 +459,25 @@ metadata: {{- include "karmada.preInstallJob.labels" . | nindent 4 }} {{- end }} --- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }}-hook-job + namespace: {{ $namespace }} + annotations: + "kubernetes.io/service-account.name": {{ $name }}-hook-job + "helm.sh/hook": pre-install + "helm.sh/hook-weight": "1" + {{- if "karmada.preInstallJob.labels" }} + labels: + {{- include "karmada.preInstallJob.labels" . | nindent 4 }} + {{- end }} +type: kubernetes.io/service-account-token +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: {{ $name }}-pre-job + name: {{ $name }}-hook-job annotations: "helm.sh/hook": pre-install "helm.sh/hook-weight": "1" @@ -481,7 +495,7 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: {{ $name }}-pre-job + name: {{ $name }}-hook-job annotations: "helm.sh/hook": pre-install "helm.sh/hook-weight": "1" @@ -492,10 +506,10 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: {{ $name }}-pre-job + name: {{ $name }}-hook-job subjects: - kind: ServiceAccount - name: {{ $name }}-pre-job + name: {{ $name }}-hook-job namespace: {{ $namespace }} --- {{- end }} diff --git a/charts/karmada/values.yaml b/charts/karmada/values.yaml index c120df909fd8..b163de1139d6 100644 --- a/charts/karmada/values.yaml +++ b/charts/karmada/values.yaml @@ -98,6 +98,11 @@ preInstallJob: ## Define policies that determine when to delete corresponding hook resources: before-hook-creation,hook-succeeded,hook-failed hookDeletePolicy: "hook-succeeded" +## static-resource job config +staticResourceJob: + tolerations: [] + nodeSelector: {} + ## post-install job config postInstallJob: tolerations: [] @@ -366,7 +371,7 @@ apiServer: image: registry: registry.k8s.io repository: kube-apiserver - tag: "v1.27.11" + tag: "v1.28.9" ## Specify a imagePullPolicy, defaults to 'IfNotPresent' pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. @@ -506,10 +511,7 @@ metricsAdapter: registry: docker.io repository: karmada/karmada-metrics-adapter tag: *karmadaImageVersion - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## - pullPolicy: Always + pullPolicy: *karmadaImagePullPolicy ## Optionally specify an array of imagePullSecrets. ## Secrets must be manually created in the namespace. ## Example: @@ -566,7 +568,7 @@ kubeControllerManager: image: registry: registry.k8s.io repository: kube-controller-manager - tag: "v1.27.11" + tag: "v1.28.9" ## Specify a imagePullPolicy, defaults to 'IfNotPresent' pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. diff --git a/cluster/images/Dockerfile b/cluster/images/Dockerfile index 455314b8febb..0282d785e000 100644 --- a/cluster/images/Dockerfile +++ b/cluster/images/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM alpine:3.20.0 +FROM alpine:3.20.1 ARG BINARY diff --git a/cluster/images/buildx.Dockerfile b/cluster/images/buildx.Dockerfile index 376107aa1b11..12584eabf0c2 100644 --- a/cluster/images/buildx.Dockerfile +++ b/cluster/images/buildx.Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM alpine:3.20.0 +FROM alpine:3.20.1 ARG BINARY ARG TARGETPLATFORM diff --git a/cmd/OWNERS b/cmd/OWNERS index 445dae621f67..f05b07fed002 100644 --- a/cmd/OWNERS +++ b/cmd/OWNERS @@ -1,4 +1,6 @@ reviewers: - lonelyCZ +- whitewindmills approvers: - lonelyCZ +- whitewindmills diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 4834b21360ce..7d4743bd2ae7 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -21,6 +21,7 @@ import ( "k8s.io/component-base/cli" _ "k8s.io/component-base/logs/json/register" // for JSON log format registration + "k8s.io/klog/v2" controllerruntime "sigs.k8s.io/controller-runtime" _ "sigs.k8s.io/controller-runtime/pkg/metrics" @@ -29,6 +30,12 @@ import ( func main() { ctx := controllerruntime.SetupSignalHandler() + // Starting from version 0.15.0, controller-runtime expects its consumers to set a logger through log.SetLogger. + // If SetLogger is not called within the first 30 seconds of a binaries lifetime, it will get + // set to a NullLogSink and report an error. Here's to silence the "log.SetLogger(...) was never called; logs will not be displayed" error + // by setting a logger through log.SetLogger. + // More info refer to: https://github.com/karmada-io/karmada/pull/4885. + controllerruntime.SetLogger(klog.Background()) cmd := app.NewAgentCommand(ctx) code := cli.Run(cmd) os.Exit(code) diff --git a/cmd/descheduler/app/options/options.go b/cmd/descheduler/app/options/options.go index 0dd0d0b816cb..67e8a54d4e34 100644 --- a/cmd/descheduler/app/options/options.go +++ b/cmd/descheduler/app/options/options.go @@ -62,6 +62,14 @@ type Options struct { SchedulerEstimatorServicePrefix string // SchedulerEstimatorPort is the port that the accurate scheduler estimator server serves at. SchedulerEstimatorPort int + // SchedulerEstimatorCertFile SSL certification file used to secure scheduler estimator communication. + SchedulerEstimatorCertFile string + // SchedulerEstimatorKeyFile SSL key file used to secure scheduler estimator communication. + SchedulerEstimatorKeyFile string + // SchedulerEstimatorCaFile SSL Certificate Authority file used to secure scheduler estimator communication. + SchedulerEstimatorCaFile string + // InsecureSkipEstimatorVerify controls whether verifies the grpc server's certificate chain and host name. + InsecureSkipEstimatorVerify bool // DeschedulingInterval specifies time interval for descheduler to run. DeschedulingInterval metav1.Duration // UnschedulableThreshold specifies the period of pod unschedulable condition. @@ -99,6 +107,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.IntVar(&o.KubeAPIBurst, "kube-api-burst", 60, "Burst to use while talking with karmada-apiserver.") fs.DurationVar(&o.SchedulerEstimatorTimeout.Duration, "scheduler-estimator-timeout", 3*time.Second, "Specifies the timeout period of calling the scheduler estimator service.") fs.IntVar(&o.SchedulerEstimatorPort, "scheduler-estimator-port", defaultEstimatorPort, "The secure port on which to connect the accurate scheduler estimator.") + fs.StringVar(&o.SchedulerEstimatorCertFile, "scheduler-estimator-cert-file", "", "SSL certification file used to secure scheduler estimator communication.") + fs.StringVar(&o.SchedulerEstimatorKeyFile, "scheduler-estimator-key-file", "", "SSL key file used to secure scheduler estimator communication.") + fs.StringVar(&o.SchedulerEstimatorCaFile, "scheduler-estimator-ca-file", "", "SSL Certificate Authority file used to secure scheduler estimator communication.") + fs.BoolVar(&o.InsecureSkipEstimatorVerify, "insecure-skip-estimator-verify", false, "Controls whether verifies the scheduler estimator's certificate chain and host name.") fs.StringVar(&o.SchedulerEstimatorServicePrefix, "scheduler-estimator-service-prefix", "karmada-scheduler-estimator", "The prefix of scheduler estimator service name") fs.DurationVar(&o.DeschedulingInterval.Duration, "descheduling-interval", defaultDeschedulingInterval, "Time interval between two consecutive descheduler executions. Setting this value instructs the descheduler to run in a continuous loop at the interval specified.") fs.DurationVar(&o.UnschedulableThreshold.Duration, "unschedulable-threshold", defaultUnschedulableThreshold, "The period of pod unschedulable condition. This value is considered as a classification standard of unschedulable replicas.") diff --git a/cmd/scheduler-estimator/app/options/options.go b/cmd/scheduler-estimator/app/options/options.go index f10bf3ca9cb2..1d6ee5d26435 100644 --- a/cmd/scheduler-estimator/app/options/options.go +++ b/cmd/scheduler-estimator/app/options/options.go @@ -40,6 +40,14 @@ type Options struct { SecurePort int // ServerPort is the port that the server gRPC serves at. ServerPort int + // InsecureSkipGrpcClientVerify controls whether verifies the grpc client's certificate chain and host name. + InsecureSkipGrpcClientVerify bool + // GrpcAuthCertFile SSL certification file used for grpc SSL/TLS connections. + GrpcAuthCertFile string + // GrpcAuthKeyFile SSL key file used for grpc SSL/TLS connections. + GrpcAuthKeyFile string + // GrpcClientCaFile SSL Certificate Authority file used to verify grpc client certificates on incoming requests. + GrpcClientCaFile string // ClusterAPIQPS is the QPS to use while talking with cluster kube-apiserver. ClusterAPIQPS float32 // ClusterAPIBurst is the burst to allow while talking with cluster kube-apiserver. @@ -64,6 +72,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&o.ClusterName, "cluster-name", o.ClusterName, "Name of member cluster that the estimator serves for.") fs.StringVar(&o.BindAddress, "bind-address", defaultBindAddress, "The IP address on which to listen for the --secure-port port.") fs.IntVar(&o.ServerPort, "server-port", defaultServerPort, "The secure port on which to serve gRPC.") + fs.StringVar(&o.GrpcAuthCertFile, "grpc-auth-cert-file", "", "SSL certification file used for grpc SSL/TLS connections.") + fs.StringVar(&o.GrpcAuthKeyFile, "grpc-auth-key-file", "", "SSL key file used for grpc SSL/TLS connections.") + fs.BoolVar(&o.InsecureSkipGrpcClientVerify, "insecure-skip-grpc-client-verify", false, "If set to true, the estimator will not verify the grpc client's certificate chain and host name. When the relevant certificates are not configured, it will not take effect.") + fs.StringVar(&o.GrpcClientCaFile, "grpc-client-ca-file", "", "SSL Certificate Authority file used to verify grpc client certificates on incoming requests if --client-cert-auth flag is set.") fs.IntVar(&o.SecurePort, "secure-port", defaultHealthzPort, "The secure port on which to serve HTTPS.") fs.Float32Var(&o.ClusterAPIQPS, "kube-api-qps", 20.0, "QPS to use while talking with apiserver.") fs.IntVar(&o.ClusterAPIBurst, "kube-api-burst", 30, "Burst to use while talking with apiserver.") diff --git a/cmd/scheduler/app/options/options.go b/cmd/scheduler/app/options/options.go index bcb6b2e59634..ac994775253c 100644 --- a/cmd/scheduler/app/options/options.go +++ b/cmd/scheduler/app/options/options.go @@ -71,6 +71,14 @@ type Options struct { SchedulerEstimatorServicePrefix string // SchedulerEstimatorPort is the port that the accurate scheduler estimator server serves at. SchedulerEstimatorPort int + // InsecureSkipEstimatorVerify controls whether verifies the grpc server's certificate chain and host name. + InsecureSkipEstimatorVerify bool + // SchedulerEstimatorCertFile SSL certification file used to secure scheduler estimator communication. + SchedulerEstimatorCertFile string + // SchedulerEstimatorKeyFile SSL key file used to secure scheduler estimator communication. + SchedulerEstimatorKeyFile string + // SchedulerEstimatorCaFile SSL Certificate Authority file used to secure scheduler estimator communication. + SchedulerEstimatorCaFile string // EnableEmptyWorkloadPropagation represents whether workload with 0 replicas could be propagated to member clusters. EnableEmptyWorkloadPropagation bool @@ -138,6 +146,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.DurationVar(&o.SchedulerEstimatorTimeout.Duration, "scheduler-estimator-timeout", 3*time.Second, "Specifies the timeout period of calling the scheduler estimator service.") fs.StringVar(&o.SchedulerEstimatorServicePrefix, "scheduler-estimator-service-prefix", "karmada-scheduler-estimator", "The prefix of scheduler estimator service name") fs.IntVar(&o.SchedulerEstimatorPort, "scheduler-estimator-port", defaultEstimatorPort, "The secure port on which to connect the accurate scheduler estimator.") + fs.StringVar(&o.SchedulerEstimatorCertFile, "scheduler-estimator-cert-file", "", "SSL certification file used to secure scheduler estimator communication.") + fs.StringVar(&o.SchedulerEstimatorKeyFile, "scheduler-estimator-key-file", "", "SSL key file used to secure scheduler estimator communication.") + fs.StringVar(&o.SchedulerEstimatorCaFile, "scheduler-estimator-ca-file", "", "SSL Certificate Authority file used to secure scheduler estimator communication.") + fs.BoolVar(&o.InsecureSkipEstimatorVerify, "insecure-skip-estimator-verify", false, "Controls whether verifies the scheduler estimator's certificate chain and host name.") fs.BoolVar(&o.EnableEmptyWorkloadPropagation, "enable-empty-workload-propagation", false, "Enable workload with replicas 0 to be propagated to member clusters.") fs.StringSliceVar(&o.Plugins, "plugins", []string{"*"}, fmt.Sprintf("A list of plugins to enable. '*' enables all build-in and customized plugins, 'foo' enables the plugin named 'foo', '*,-foo' disables the plugin named 'foo'.\nAll build-in plugins: %s.", strings.Join(frameworkplugins.NewInTreeRegistry().FactoryNames(), ","))) diff --git a/cmd/scheduler/app/scheduler.go b/cmd/scheduler/app/scheduler.go index 826adbf99a0c..9d4bad259136 100644 --- a/cmd/scheduler/app/scheduler.go +++ b/cmd/scheduler/app/scheduler.go @@ -170,7 +170,7 @@ func run(opts *options.Options, stopChan <-chan struct{}, registryOptions ...Opt scheduler.WithEnableSchedulerEstimator(opts.EnableSchedulerEstimator), scheduler.WithDisableSchedulerEstimatorInPullMode(opts.DisableSchedulerEstimatorInPullMode), scheduler.WithSchedulerEstimatorServicePrefix(opts.SchedulerEstimatorServicePrefix), - scheduler.WithSchedulerEstimatorPort(opts.SchedulerEstimatorPort), + scheduler.WithSchedulerEstimatorConnection(opts.SchedulerEstimatorPort, opts.SchedulerEstimatorCertFile, opts.SchedulerEstimatorKeyFile, opts.SchedulerEstimatorCaFile, opts.InsecureSkipEstimatorVerify), scheduler.WithSchedulerEstimatorTimeout(opts.SchedulerEstimatorTimeout), scheduler.WithEnableEmptyWorkloadPropagation(opts.EnableEmptyWorkloadPropagation), scheduler.WithEnableSchedulerPlugin(opts.Plugins), diff --git a/docs/CHANGELOG/CHANGELOG-1.10.md b/docs/CHANGELOG/CHANGELOG-1.10.md index 89e7f24a054d..2961087633b9 100644 --- a/docs/CHANGELOG/CHANGELOG-1.10.md +++ b/docs/CHANGELOG/CHANGELOG-1.10.md @@ -2,6 +2,18 @@ **Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* +- [v1.10.2](#v1102) + - [Downloads for v1.10.2](#downloads-for-v1102) + - [Changelog since v1.10.1](#changelog-since-v1101) + - [Changes by Kind](#changes-by-kind) + - [Bug Fixes](#bug-fixes) + - [Others](#others) +- [v1.10.1](#v1101) + - [Downloads for v1.10.1](#downloads-for-v1101) + - [Changelog since v1.10.0](#changelog-since-v1100) + - [Changes by Kind](#changes-by-kind-1) + - [Bug Fixes](#bug-fixes-1) + - [Others](#others-1) - [v1.10.0](#v1100) - [Downloads for v1.10.0](#downloads-for-v1100) - [What's New](#whats-new) @@ -10,7 +22,7 @@ - [Other Notable Changes](#other-notable-changes) - [API Changes](#api-changes) - [Deprecation](#deprecation) - - [Bug Fixes](#bug-fixes) + - [Bug Fixes](#bug-fixes-2) - [Security](#security) - [Features & Enhancements](#features--enhancements) - [Other](#other) @@ -21,6 +33,33 @@ +# v1.10.2 +## Downloads for v1.10.2 + +Download v1.10.2 in the [v1.10.2 release page](https://github.com/karmada-io/karmada/releases/tag/v1.10.2). + +## Changelog since v1.10.1 +### Changes by Kind +#### Bug Fixes +- `karmada-controller-manager`: Fixed the issue that the default resource interpreter doesn't accurately interpret the numbers of replicas. ([#5108](https://github.com/karmada-io/karmada/pull/5108), @whitewindmills) + +#### Others +- The base image `alpine` now has been promoted from `alpine:3.20.0` to `alpine:3.20.1`. ([#5093](https://github.com/karmada-io/karmada/pull/5093)) + +# v1.10.1 +## Downloads for v1.10.1 + +Download v1.10.1 in the [v1.10.1 release page](https://github.com/karmada-io/karmada/releases/tag/v1.10.1). + +## Changelog since v1.10.0 +### Changes by Kind +#### Bug Fixes +- `karmada-scheduler-estimator`: Fixed the `Unschedulable` result returned by plugins to be treated as an exception issue. ([#5027](https://github.com/karmada-io/karmada/pull/5027), @RainbowMango) +- `karmada-controller-manager`: Fixed an issue that the cluster-status-controller overwrites the remedyActions field. ([#5043](https://github.com/karmada-io/karmada/pull/5043), @XiShanYongYe-Chang) + +#### Others +None. + # v1.10.0 ## Downloads for v1.10.0 diff --git a/docs/CHANGELOG/CHANGELOG-1.8.md b/docs/CHANGELOG/CHANGELOG-1.8.md index 5ce914be0839..3887e9297e26 100644 --- a/docs/CHANGELOG/CHANGELOG-1.8.md +++ b/docs/CHANGELOG/CHANGELOG-1.8.md @@ -2,30 +2,36 @@ **Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* -- [v1.8.4](#v184) - - [Downloads for v1.8.4](#downloads-for-v184) - - [Changelog since v1.8.3](#changelog-since-v183) +- [v1.8.5](#v185) + - [Downloads for v1.8.5](#downloads-for-v185) + - [Changelog since v1.8.4](#changelog-since-v184) - [Changes by Kind](#changes-by-kind) - [Bug Fixes](#bug-fixes) - [Others](#others) -- [v1.8.3](#v183) - - [Downloads for v1.8.3](#downloads-for-v183) - - [Changelog since v1.8.2](#changelog-since-v182) +- [v1.8.4](#v184) + - [Downloads for v1.8.4](#downloads-for-v184) + - [Changelog since v1.8.3](#changelog-since-v183) - [Changes by Kind](#changes-by-kind-1) - [Bug Fixes](#bug-fixes-1) - [Others](#others-1) -- [v1.8.2](#v182) - - [Downloads for v1.8.2](#downloads-for-v182) - - [Changelog since v1.8.1](#changelog-since-v181) +- [v1.8.3](#v183) + - [Downloads for v1.8.3](#downloads-for-v183) + - [Changelog since v1.8.2](#changelog-since-v182) - [Changes by Kind](#changes-by-kind-2) - [Bug Fixes](#bug-fixes-2) - [Others](#others-2) -- [v1.8.1](#v181) - - [Downloads for v1.8.1](#downloads-for-v181) - - [Changelog since v1.8.0](#changelog-since-v180) +- [v1.8.2](#v182) + - [Downloads for v1.8.2](#downloads-for-v182) + - [Changelog since v1.8.1](#changelog-since-v181) - [Changes by Kind](#changes-by-kind-3) - [Bug Fixes](#bug-fixes-3) - [Others](#others-3) +- [v1.8.1](#v181) + - [Downloads for v1.8.1](#downloads-for-v181) + - [Changelog since v1.8.0](#changelog-since-v180) + - [Changes by Kind](#changes-by-kind-4) + - [Bug Fixes](#bug-fixes-4) + - [Others](#others-4) - [v1.8.0](#v180) - [Downloads for v1.8.0](#downloads-for-v180) - [What's New](#whats-new) @@ -36,7 +42,7 @@ - [Other Notable Changes](#other-notable-changes) - [API Changes](#api-changes) - [Deprecation](#deprecation) - - [Bug Fixes](#bug-fixes-4) + - [Bug Fixes](#bug-fixes-5) - [Security](#security) - [Features & Enhancements](#features--enhancements) - [Other](#other) @@ -47,6 +53,19 @@ +# v1.8.5 +## Downloads for v1.8.5 + +Download v1.8.5 in the [v1.8.5 release page](https://github.com/karmada-io/karmada/releases/tag/v1.8.5). + +## Changelog since v1.8.4 +### Changes by Kind +#### Bug Fixes +- `karmada-controller-manager`: Fixed the issue that the default resource interpreter doesn't accurately interpret the numbers of replicas. ([#5106](https://github.com/karmada-io/karmada/pull/5108), @whitewindmills) + +#### Others +- The base image `alpine` now has been promoted from `alpine:3.20.0` to `alpine:3.20.1`. ([#5088](https://github.com/karmada-io/karmada/pull/5093)) + # v1.8.4 ## Downloads for v1.8.4 diff --git a/docs/CHANGELOG/CHANGELOG-1.9.md b/docs/CHANGELOG/CHANGELOG-1.9.md index 313d0f30ca80..63cd94b41ca2 100644 --- a/docs/CHANGELOG/CHANGELOG-1.9.md +++ b/docs/CHANGELOG/CHANGELOG-1.9.md @@ -2,18 +2,30 @@ **Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* -- [v1.9.2](#v192) - - [Downloads for v1.9.2](#downloads-for-v192) - - [Changelog since v1.9.1](#changelog-since-v191) +- [v1.9.4](#v194) + - [Downloads for v1.9.4](#downloads-for-v194) + - [Changelog since v1.9.3](#changelog-since-v193) - [Changes by Kind](#changes-by-kind) - [Bug Fixes](#bug-fixes) - [Others](#others) -- [v1.9.1](#v191) - - [Downloads for v1.9.1](#downloads-for-v191) - - [Changelog since v1.9.0](#changelog-since-v190) +- [v1.9.3](#v193) + - [Downloads for v1.9.3](#downloads-for-v193) + - [Changelog since v1.9.2](#changelog-since-v192) - [Changes by Kind](#changes-by-kind-1) - [Bug Fixes](#bug-fixes-1) - [Others](#others-1) +- [v1.9.2](#v192) + - [Downloads for v1.9.2](#downloads-for-v192) + - [Changelog since v1.9.1](#changelog-since-v191) + - [Changes by Kind](#changes-by-kind-2) + - [Bug Fixes](#bug-fixes-2) + - [Others](#others-2) +- [v1.9.1](#v191) + - [Downloads for v1.9.1](#downloads-for-v191) + - [Changelog since v1.9.0](#changelog-since-v190) + - [Changes by Kind](#changes-by-kind-3) + - [Bug Fixes](#bug-fixes-3) + - [Others](#others-3) - [v1.9.0](#v190) - [Downloads for v1.9.0](#downloads-for-v190) - [What's New](#whats-new) @@ -23,7 +35,7 @@ - [Other Notable Changes](#other-notable-changes) - [API Changes](#api-changes) - [Deprecation](#deprecation) - - [Bug Fixes](#bug-fixes-2) + - [Bug Fixes](#bug-fixes-4) - [Security](#security) - [Features & Enhancements](#features--enhancements) - [Other](#other) @@ -34,6 +46,33 @@ +# v1.9.4 +## Downloads for v1.9.4 + +Download v1.9.4 in the [v1.9.4 release page](https://github.com/karmada-io/karmada/releases/tag/v1.9.4). + +## Changelog since v1.9.3 +### Changes by Kind +#### Bug Fixes +- `karmada-controller-manager`: Fixed the issue that the default resource interpreter doesn't accurately interpret the numbers of replicas. ([#5107](https://github.com/karmada-io/karmada/pull/5107), @whitewindmills) + +#### Others +- The base image `alpine` now has been promoted from `alpine:3.20.0` to `alpine:3.20.1`. ([#5089](https://github.com/karmada-io/karmada/pull/5089)) + +# v1.9.3 +## Downloads for v1.9.3 + +Download v1.9.3 in the [v1.9.3 release page](https://github.com/karmada-io/karmada/releases/tag/v1.9.3). + +## Changelog since v1.9.2 +### Changes by Kind +#### Bug Fixes +- `karmada-scheduler-estimator`: Fixed the `Unschedulable` result returned by plugins to be treated as an exception issue. ([#5026](https://github.com/karmada-io/karmada/pull/5026), @RainbowMango) +- `karmada-controller-manager`: Fixed an issue that the cluster-status-controller overwrites the remedyActions field. ([#5045](https://github.com/karmada-io/karmada/pull/5045), @XiShanYongYe-Chang) + +#### Others +None. + # v1.9.2 ## Downloads for v1.9.2 diff --git a/docs/images/karmada-resource-relation.png b/docs/images/karmada-resource-relation.png index 53b99c9ea28f..7b440e50f86f 100644 Binary files a/docs/images/karmada-resource-relation.png and b/docs/images/karmada-resource-relation.png differ diff --git a/hack/deploy-karmada.sh b/hack/deploy-karmada.sh index 97989c6a738e..b4cd1af2c595 100755 --- a/hack/deploy-karmada.sh +++ b/hack/deploy-karmada.sh @@ -200,8 +200,10 @@ fi # deploy karmada apiserver TEMP_PATH_APISERVER=$(mktemp -d) trap '{ rm -rf ${TEMP_PATH_APISERVER}; }' EXIT +KARMADA_APISERVER_VERSION=${KARMADA_APISERVER_VERSION:-"v1.28.9"} cp "${REPO_ROOT}"/artifacts/deploy/karmada-apiserver.yaml "${TEMP_PATH_APISERVER}"/karmada-apiserver.yaml sed -i'' -e "s/{{service_type}}/${KARMADA_APISERVER_SERVICE_TYPE}/g" "${TEMP_PATH_APISERVER}"/karmada-apiserver.yaml +sed -i'' -e "s/{{karmada_apiserver_version}}/${KARMADA_APISERVER_VERSION}/g" "${TEMP_PATH_APISERVER}"/karmada-apiserver.yaml echo -e "\nApply dynamic rendered apiserver service in ${TEMP_PATH_APISERVER}/karmada-apiserver.yaml." kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${TEMP_PATH_APISERVER}"/karmada-apiserver.yaml @@ -238,7 +240,9 @@ fi util::append_client_kubeconfig "${HOST_CLUSTER_KUBECONFIG}" "${CERT_DIR}/karmada.crt" "${CERT_DIR}/karmada.key" "${KARMADA_APISERVER_IP}" "${KARMADA_APISERVER_SECURE_PORT}" karmada-apiserver # deploy kube controller manager -kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${REPO_ROOT}/artifacts/deploy/kube-controller-manager.yaml" +cp "${REPO_ROOT}"/artifacts/deploy/kube-controller-manager.yaml "${TEMP_PATH_APISERVER}"/kube-controller-manager.yaml +sed -i'' -e "s/{{karmada_apiserver_version}}/${KARMADA_APISERVER_VERSION}/g" "${TEMP_PATH_APISERVER}"/kube-controller-manager.yaml +kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${TEMP_PATH_APISERVER}"/kube-controller-manager.yaml # deploy aggregated-apiserver on host cluster kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${REPO_ROOT}/artifacts/deploy/karmada-aggregated-apiserver.yaml" util::wait_pod_ready "${HOST_CLUSTER_NAME}" "${KARMADA_AGGREGATION_APISERVER_LABEL}" "${KARMADA_SYSTEM_NAMESPACE}" diff --git a/hack/local-up-karmada.sh b/hack/local-up-karmada.sh index 867b0c5eb91c..17336e6e7c53 100755 --- a/hack/local-up-karmada.sh +++ b/hack/local-up-karmada.sh @@ -111,6 +111,7 @@ trap '{ rm -rf ${TEMP_PATH}; }' EXIT echo -e "Preparing kindClusterConfig in path: ${TEMP_PATH}" cp -rf "${REPO_ROOT}"/artifacts/kindClusterConfig/member1.yaml "${TEMP_PATH}"/member1.yaml cp -rf "${REPO_ROOT}"/artifacts/kindClusterConfig/member2.yaml "${TEMP_PATH}"/member2.yaml +cp -rf "${REPO_ROOT}"/artifacts/kindClusterConfig/member3.yaml "${TEMP_PATH}"/member3.yaml util::delete_necessary_resources "${MAIN_KUBECONFIG},${MEMBER_CLUSTER_KUBECONFIG}" "${HOST_CLUSTER_NAME},${MEMBER_CLUSTER_1_NAME},${MEMBER_CLUSTER_2_NAME},${PULL_MODE_CLUSTER_NAME}" "${KIND_LOG_FILE}" @@ -120,13 +121,14 @@ if [[ -n "${HOST_IPADDRESS}" ]]; then # If bind the port of clusters(karmada-hos sed -i'' -e "s/{{host_ipaddress}}/${HOST_IPADDRESS}/g" "${TEMP_PATH}"/karmada-host.yaml sed -i'' -e 's/networking:/&\'$'\n'' apiServerAddress: "'${HOST_IPADDRESS}'"/' "${TEMP_PATH}"/member1.yaml sed -i'' -e 's/networking:/&\'$'\n'' apiServerAddress: "'${HOST_IPADDRESS}'"/' "${TEMP_PATH}"/member2.yaml + sed -i'' -e 's/networking:/&\'$'\n'' apiServerAddress: "'${HOST_IPADDRESS}'"/' "${TEMP_PATH}"/member3.yaml util::create_cluster "${HOST_CLUSTER_NAME}" "${MAIN_KUBECONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" "${TEMP_PATH}"/karmada-host.yaml else util::create_cluster "${HOST_CLUSTER_NAME}" "${MAIN_KUBECONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" fi util::create_cluster "${MEMBER_CLUSTER_1_NAME}" "${MEMBER_CLUSTER_1_TMP_CONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" "${TEMP_PATH}"/member1.yaml util::create_cluster "${MEMBER_CLUSTER_2_NAME}" "${MEMBER_CLUSTER_2_TMP_CONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" "${TEMP_PATH}"/member2.yaml -util::create_cluster "${PULL_MODE_CLUSTER_NAME}" "${PULL_MODE_CLUSTER_TMP_CONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" +util::create_cluster "${PULL_MODE_CLUSTER_NAME}" "${PULL_MODE_CLUSTER_TMP_CONFIG}" "${CLUSTER_VERSION}" "${KIND_LOG_FILE}" "${TEMP_PATH}"/member3.yaml #step2. make images and get karmadactl export VERSION="latest" diff --git a/hack/tools/genkarmadactldocs/gen_karmadactl_docs.go b/hack/tools/genkarmadactldocs/gen_karmadactl_docs.go index f7b962024d11..5a4dbce77335 100644 --- a/hack/tools/genkarmadactldocs/gen_karmadactl_docs.go +++ b/hack/tools/genkarmadactldocs/gen_karmadactl_docs.go @@ -111,15 +111,15 @@ func main() { // Set environment variables used by karmadactl so the output is consistent, // regardless of where we run. os.Setenv("HOME", "/home/username") - karmadactl := karmadactl.NewKarmadaCtlCommand("karmadactl", "karmadactl") - karmadactl.DisableAutoGenTag = true - err = doc.GenMarkdownTree(karmadactl, outDir) + karmadactlCmd := karmadactl.NewKarmadaCtlCommand("karmadactl", "karmadactl") + karmadactlCmd.DisableAutoGenTag = true + err = doc.GenMarkdownTree(karmadactlCmd, outDir) if err != nil { fmt.Fprintf(os.Stderr, "failed to generate docs: %v\n", err) os.Exit(1) } - err = GenMarkdownTreeForIndex(karmadactl, outDir) + err = GenMarkdownTreeForIndex(karmadactlCmd, outDir) if err != nil { fmt.Fprintf(os.Stderr, "failed to generate index docs: %v\n", err) os.Exit(1) @@ -153,8 +153,8 @@ func main() { newlines := []string{"---", "title: " + title} newlines = append(newlines, lines...) - newcontent := strings.Join(newlines, "\n") - return os.WriteFile(path, []byte(newcontent), info.Mode()) + newContent := strings.Join(newlines, "\n") + return os.WriteFile(path, []byte(newContent), info.Mode()) }) if err != nil { fmt.Fprintf(os.Stderr, "failed to process docs: %v\n", err) diff --git a/hack/tools/tools.go b/hack/tools/tools.go index 969b77195bf0..89dac2b1100e 100644 --- a/hack/tools/tools.go +++ b/hack/tools/tools.go @@ -25,4 +25,5 @@ import ( _ "go.uber.org/mock/mockgen" _ "golang.org/x/tools/cmd/goimports" _ "k8s.io/code-generator" + _ "k8s.io/kube-openapi/cmd/openapi-gen" ) diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index 9fb316ff56e6..da6050432fc2 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -28,7 +28,7 @@ GO111MODULE=on go install k8s.io/code-generator/cmd/conversion-gen GO111MODULE=on go install k8s.io/code-generator/cmd/client-gen GO111MODULE=on go install k8s.io/code-generator/cmd/lister-gen GO111MODULE=on go install k8s.io/code-generator/cmd/informer-gen -GO111MODULE=on go install k8s.io/code-generator/cmd/openapi-gen +GO111MODULE=on go install k8s.io/kube-openapi/cmd/openapi-gen export GOPATH=$(go env GOPATH | awk -F ':' '{print $1}') export PATH=$PATH:$GOPATH/bin diff --git a/hack/update-estimator-protobuf.sh b/hack/update-estimator-protobuf.sh index 8ce6b2bd21f7..b11b73f20440 100755 --- a/hack/update-estimator-protobuf.sh +++ b/hack/update-estimator-protobuf.sh @@ -59,12 +59,12 @@ PACKAGES=( ) APIMACHINERY_PKGS=( - +k8s.io/apimachinery/pkg/util/intstr - +k8s.io/apimachinery/pkg/api/resource - +k8s.io/apimachinery/pkg/runtime/schema - +k8s.io/apimachinery/pkg/runtime - k8s.io/apimachinery/pkg/apis/meta/v1 - k8s.io/api/core/v1 + -k8s.io/apimachinery/pkg/util/intstr + -k8s.io/apimachinery/pkg/api/resource + -k8s.io/apimachinery/pkg/runtime/schema + -k8s.io/apimachinery/pkg/runtime + -k8s.io/apimachinery/pkg/apis/meta/v1 + -k8s.io/api/core/v1 ) go-to-protobuf \ @@ -72,10 +72,7 @@ go-to-protobuf \ --apimachinery-packages=$(IFS=, ; echo "${APIMACHINERY_PKGS[*]}") \ --packages=$(IFS=, ; echo "${PACKAGES[*]}") \ --proto-import="${KARMADA_ROOT}/vendor" \ - --proto-import="${KARMADA_ROOT}/third_party/protobuf" + --proto-import="${KARMADA_ROOT}/third_party/protobuf" \ + --output-base="${GOPATH}/src" go generate ./pkg/estimator/service - -# The `go-to-protobuf` tool will modify all import proto files in vendor, so we should use go mod vendor to prevent. -export GOPATH=${DEFAULT_GOPATH} -go mod vendor diff --git a/operator/README.md b/operator/README.md index 14fc3cd27c2e..de2d6fd0a226 100644 --- a/operator/README.md +++ b/operator/README.md @@ -211,7 +211,7 @@ You can change it to `NodePort`: ... karmadaAPIServer: imageRepository: registry.k8s.io/kube-apiserver - imageTag: v1.27.11 + imageTag: v1.28.9 replicas: 1 serviceType: NodePort serviceSubnet: 10.96.0.0/12 @@ -226,7 +226,7 @@ You can add more SANs to karmada-apiserver certificate: ... karmadaAPIServer: imageRepository: registry.k8s.io/kube-apiserver - imageTag: v1.27.11 + imageTag: v1.28.9 replicas: 1 serviceSubnet: 10.96.0.0/12 certSANs: diff --git a/operator/cmd/operator/operator.go b/operator/cmd/operator/operator.go index 5d0d28a366d5..60abfde88e2e 100644 --- a/operator/cmd/operator/operator.go +++ b/operator/cmd/operator/operator.go @@ -21,6 +21,7 @@ import ( "k8s.io/component-base/cli" _ "k8s.io/component-base/logs/json/register" // for JSON log format registration + "k8s.io/klog/v2" controllerruntime "sigs.k8s.io/controller-runtime" _ "sigs.k8s.io/controller-runtime/pkg/metrics" @@ -29,6 +30,12 @@ import ( func main() { ctx := controllerruntime.SetupSignalHandler() + // Starting from version 0.15.0, controller-runtime expects its consumers to set a logger through log.SetLogger. + // If SetLogger is not called within the first 30 seconds of a binaries lifetime, it will get + // set to a NullLogSink and report an error. Here's to silence the "log.SetLogger(...) was never called; logs will not be displayed" error + // by setting a logger through log.SetLogger. + // More info refer to: https://github.com/karmada-io/karmada/pull/4885. + controllerruntime.SetLogger(klog.Background()) command := app.NewOperatorCommand(ctx) code := cli.Run(command) os.Exit(code) diff --git a/operator/config/crds/operator.karmada.io_karmadas.yaml b/operator/config/crds/operator.karmada.io_karmadas.yaml index f919ca2b94d1..bed6b2796fb7 100644 --- a/operator/config/crds/operator.karmada.io_karmadas.yaml +++ b/operator/config/crds/operator.karmada.io_karmadas.yaml @@ -18,10 +18,10 @@ spec: versions: - additionalPrinterColumns: - jsonPath: .status.conditions[?(@.type=="Ready")].status - name: Ready + name: READY type: string - jsonPath: .metadata.creationTimestamp - name: Age + name: AGE type: date name: v1alpha1 schema: diff --git a/operator/config/samples/karmada.yaml b/operator/config/samples/karmada.yaml index d315842fe263..8de6581a1fc2 100644 --- a/operator/config/samples/karmada.yaml +++ b/operator/config/samples/karmada.yaml @@ -25,7 +25,7 @@ spec: storage: 3Gi karmadaAPIServer: imageRepository: registry.k8s.io/kube-apiserver - imageTag: v1.27.11 + imageTag: v1.28.9 replicas: 1 serviceType: NodePort serviceSubnet: 10.96.0.0/12 @@ -47,7 +47,7 @@ spec: replicas: 1 kubeControllerManager: imageRepository: registry.k8s.io/kube-controller-manager - imageTag: v1.27.11 + imageTag: v1.28.9 replicas: 1 karmadaMetricsAdapter: imageRepository: docker.io/karmada/karmada-metrics-adapter diff --git a/operator/pkg/apis/operator/v1alpha1/type.go b/operator/pkg/apis/operator/v1alpha1/type.go index ff16bd85cc2d..fbe5dfd72711 100644 --- a/operator/pkg/apis/operator/v1alpha1/type.go +++ b/operator/pkg/apis/operator/v1alpha1/type.go @@ -25,8 +25,8 @@ import ( // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:subresource:status // +kubebuilder:resource:path=karmadas,scope=Namespaced,categories={karmada-io} -// +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="Ready")].status`,name="Ready",type=string -// +kubebuilder:printcolumn:JSONPath=`.metadata.creationTimestamp`,name="Age",type=date +// +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="Ready")].status`,name="READY",type=string +// +kubebuilder:printcolumn:JSONPath=`.metadata.creationTimestamp`,name="AGE",type=date // Karmada enables declarative installation of karmada. type Karmada struct { diff --git a/operator/pkg/certs/certs.go b/operator/pkg/certs/certs.go index 798f36a3f036..d58c4db3415a 100644 --- a/operator/pkg/certs/certs.go +++ b/operator/pkg/certs/certs.go @@ -347,10 +347,6 @@ func NewSignedCert(cc *CertConfig, key crypto.Signer, caCert *x509.Certificate, } RemoveDuplicateAltNames(&cc.Config.AltNames) - notAfter := time.Now().Add(constants.CertificateValidity).UTC() - if cc.NotAfter != nil { - notAfter = *cc.NotAfter - } certTmpl := x509.Certificate{ Subject: pkix.Name{ @@ -361,7 +357,7 @@ func NewSignedCert(cc *CertConfig, key crypto.Signer, caCert *x509.Certificate, IPAddresses: cc.Config.AltNames.IPs, SerialNumber: serial, NotBefore: caCert.NotBefore, - NotAfter: notAfter, + NotAfter: cc.NotAfter.UTC(), KeyUsage: keyUsage, ExtKeyUsage: cc.Config.Usages, BasicConstraintsValid: true, diff --git a/operator/pkg/constants/constants.go b/operator/pkg/constants/constants.go index 22143347af4e..b85ab1c65218 100644 --- a/operator/pkg/constants/constants.go +++ b/operator/pkg/constants/constants.go @@ -31,7 +31,7 @@ const ( // EtcdDefaultVersion defines the default of the karmada etcd image tag EtcdDefaultVersion = "3.5.13-0" // KubeDefaultVersion defines the default of the karmada apiserver and kubeControllerManager image tag - KubeDefaultVersion = "v1.27.11" + KubeDefaultVersion = "v1.28.9" // KarmadaDefaultServiceSubnet defines the default of the subnet used by k8s services. KarmadaDefaultServiceSubnet = "10.96.0.0/12" // KarmadaDefaultDNSDomain defines the default of the DNSDomain diff --git a/operator/pkg/controlplane/manifests.go b/operator/pkg/controlplane/manifests.go index e7480b91d25d..fc67c4d2fa96 100644 --- a/operator/pkg/controlplane/manifests.go +++ b/operator/pkg/controlplane/manifests.go @@ -188,6 +188,9 @@ spec: - --secure-port=10351 - --enable-scheduler-estimator=true - --leader-elect-resource-namespace={{ .SystemNamespace }} + - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt + - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt + - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key - --v=4 livenessProbe: httpGet: @@ -199,10 +202,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: karmada-certs + mountPath: /etc/karmada/pki + readOnly: true - name: kubeconfig subPath: kubeconfig mountPath: /etc/karmada/kubeconfig volumes: + - name: karmada-certs + secret: + secretName: {{ .KarmadaCertsSecret }} - name: kubeconfig secret: secretName: {{ .KubeconfigSecret }} @@ -241,6 +250,9 @@ spec: - --kubeconfig=/etc/karmada/kubeconfig - --bind-address=0.0.0.0 - --leader-elect-resource-namespace={{ .SystemNamespace }} + - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt + - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt + - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key - --v=4 livenessProbe: httpGet: @@ -252,10 +264,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: karmada-certs + mountPath: /etc/karmada/pki + readOnly: true - name: kubeconfig subPath: kubeconfig mountPath: /etc/karmada/kubeconfig volumes: + - name: karmada-certs + secret: + secretName: {{ .KarmadaCertsSecret }} - name: kubeconfig secret: secretName: {{ .KubeconfigSecret }} diff --git a/pkg/apis/OWNERS b/pkg/apis/OWNERS new file mode 100644 index 000000000000..43db0cc435f3 --- /dev/null +++ b/pkg/apis/OWNERS @@ -0,0 +1,12 @@ +reviewers: +- chaunceyjiang +- Garrybest +- kevin-wangzefeng +- RainbowMango +- whitewindmills +- XiShanYongYe-Chang +approvers: +- kevin-wangzefeng +- RainbowMango +options: + no_parent_owners: true diff --git a/pkg/apis/autoscaling/v1alpha1/cronfederatedhpa_types.go b/pkg/apis/autoscaling/v1alpha1/cronfederatedhpa_types.go index 0ae1e1594e94..b1ff38ad9c8a 100644 --- a/pkg/apis/autoscaling/v1alpha1/cronfederatedhpa_types.go +++ b/pkg/apis/autoscaling/v1alpha1/cronfederatedhpa_types.go @@ -1,9 +1,12 @@ /* Copyright 2023 The Karmada Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/apis/config/v1alpha1/resourceinterpretercustomization_types.go b/pkg/apis/config/v1alpha1/resourceinterpretercustomization_types.go index 384613dd5d85..4da59aeedbab 100644 --- a/pkg/apis/config/v1alpha1/resourceinterpretercustomization_types.go +++ b/pkg/apis/config/v1alpha1/resourceinterpretercustomization_types.go @@ -36,6 +36,9 @@ const ( // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:resource:path=resourceinterpretercustomizations,scope="Cluster",shortName=ric,categories={karmada-io} // +kubebuilder:storageversion +// +kubebuilder:printcolumn:JSONPath=`.spec.target.apiVersion`,name="TARGET-API-VERSION",type=string +// +kubebuilder:printcolumn:JSONPath=`.spec.target.kind`,name="TARGET-KIND",type=string +// +kubebuilder:printcolumn:JSONPath=`.metadata.creationTimestamp`,name="AGE",type=date // ResourceInterpreterCustomization describes the configuration of a specific // resource for Karmada to get the structure. diff --git a/pkg/apis/policy/v1alpha1/propagation_types.go b/pkg/apis/policy/v1alpha1/propagation_types.go index 89f196883642..f55c99dc3796 100644 --- a/pkg/apis/policy/v1alpha1/propagation_types.go +++ b/pkg/apis/policy/v1alpha1/propagation_types.go @@ -44,6 +44,9 @@ const ( // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:resource:path=propagationpolicies,scope=Namespaced,shortName=pp,categories={karmada-io} +// +kubebuilder:printcolumn:JSONPath=`.spec.conflictResolution`,name="CONFLICT-RESOLUTION",type=string +// +kubebuilder:printcolumn:JSONPath=`.spec.priority`,name="PRIORITY",type=string +// +kubebuilder:printcolumn:JSONPath=`.metadata.creationTimestamp`,name="AGE",type=date // PropagationPolicy represents the policy that propagates a group of resources to one or more clusters. type PropagationPolicy struct { @@ -562,6 +565,9 @@ type PropagationPolicyList struct { // +genclient:nonNamespaced // +kubebuilder:resource:path=clusterpropagationpolicies,scope="Cluster",shortName=cpp,categories={karmada-io} // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:printcolumn:JSONPath=`.spec.conflictResolution`,name="CONFLICT-RESOLUTION",type=string +// +kubebuilder:printcolumn:JSONPath=`.spec.priority`,name="PRIORITY",type=string +// +kubebuilder:printcolumn:JSONPath=`.metadata.creationTimestamp`,name="AGE",type=date // ClusterPropagationPolicy represents the cluster-wide policy that propagates a group of resources to one or more clusters. // Different with PropagationPolicy that could only propagate resources in its own namespace, ClusterPropagationPolicy diff --git a/pkg/apis/work/v1alpha1/work_types.go b/pkg/apis/work/v1alpha1/work_types.go index 3c3b2ed47032..c90289b2ac8f 100644 --- a/pkg/apis/work/v1alpha1/work_types.go +++ b/pkg/apis/work/v1alpha1/work_types.go @@ -36,9 +36,9 @@ const ( // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:subresource:status // +kubebuilder:resource:path=works,scope=Namespaced,shortName=wk,categories={karmada-io} -// +kubebuilder:printcolumn:JSONPath=`.spec.workload.manifests[*].kind`,name="Workload-Kind",type=string -// +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="Applied")].status`,name="Applied",type=string -// +kubebuilder:printcolumn:JSONPath=`.metadata.creationTimestamp`,name="Age",type=date +// +kubebuilder:printcolumn:JSONPath=`.spec.workload.manifests[*].kind`,name="WORKLOAD-KIND",type=string +// +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="Applied")].status`,name="APPLIED",type=string +// +kubebuilder:printcolumn:JSONPath=`.metadata.creationTimestamp`,name="AGE",type=date // Work defines a list of resources to be deployed on the member cluster. type Work struct { diff --git a/pkg/apis/work/v1alpha2/binding_types.go b/pkg/apis/work/v1alpha2/binding_types.go index 0c16b71d36ad..347a5682944f 100644 --- a/pkg/apis/work/v1alpha2/binding_types.go +++ b/pkg/apis/work/v1alpha2/binding_types.go @@ -50,9 +50,9 @@ const ( // +kubebuilder:subresource:status // +kubebuilder:resource:path=resourcebindings,scope=Namespaced,shortName=rb,categories={karmada-io} // +kubebuilder:storageversion -// +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="Scheduled")].status`,name="Scheduled",type=string -// +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="FullyApplied")].status`,name="FullyApplied",type=string -// +kubebuilder:printcolumn:JSONPath=`.metadata.creationTimestamp`,name="Age",type=date +// +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="Scheduled")].status`,name="SCHEDULED",type=string +// +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="FullyApplied")].status`,name="FULLYAPPLIED",type=string +// +kubebuilder:printcolumn:JSONPath=`.metadata.creationTimestamp`,name="AGE",type=date // ResourceBinding represents a binding of a kubernetes resource with a propagation policy. type ResourceBinding struct { @@ -409,9 +409,9 @@ const ( // +kubebuilder:resource:path=clusterresourcebindings,scope="Cluster",shortName=crb,categories={karmada-io} // +kubebuilder:subresource:status // +kubebuilder:storageversion -// +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="Scheduled")].status`,name="Scheduled",type=string -// +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="FullyApplied")].status`,name="FullyApplied",type=string -// +kubebuilder:printcolumn:JSONPath=`.metadata.creationTimestamp`,name="Age",type=date +// +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="Scheduled")].status`,name="SCHEDULED",type=string +// +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="FullyApplied")].status`,name="FULLYAPPLIED",type=string +// +kubebuilder:printcolumn:JSONPath=`.metadata.creationTimestamp`,name="AGE",type=date // ClusterResourceBinding represents a binding of a kubernetes resource with a ClusterPropagationPolicy. type ClusterResourceBinding struct { diff --git a/pkg/apis/work/v1alpha2/well_known_constants.go b/pkg/apis/work/v1alpha2/well_known_constants.go index 3aa122b79031..746b318a3e23 100644 --- a/pkg/apis/work/v1alpha2/well_known_constants.go +++ b/pkg/apis/work/v1alpha2/well_known_constants.go @@ -60,6 +60,17 @@ const ( // BindingManagedByLabel is added to ResourceBinding to represent what kind of resource manages this Binding. BindingManagedByLabel = "binding.karmada.io/managed-by" + + // ResourceTemplateGenerationAnnotationKey records the generation of resource template in Karmada APIServer, + // It will be injected into the resource when propagating to member clusters, to denote the specific version of + // the resource template from which the resource is derived. It might be helpful in the following cases: + // 1. Facilitating observation from member clusters to ascertain if the most recent resource template has been + // completely synced. + // 2. The annotation will be synced back to Karmada during the process of syncing resource status, + // by leveraging this annotation, Karmada can infer if the most recent resource template has been completely + // synced on member clusters, then generates accurate observed generation(like Deployment's .status.observedGeneration) + // which might be required by the release system. + ResourceTemplateGenerationAnnotationKey = "resourcetemplate.karmada.io/generation" ) // Define resource conflict resolution diff --git a/pkg/controllers/OWNERS b/pkg/controllers/OWNERS index 133091cd1451..adccb4a98f9f 100644 --- a/pkg/controllers/OWNERS +++ b/pkg/controllers/OWNERS @@ -10,4 +10,5 @@ reviewers: approvers: - chaunceyjiang - Garrybest +- whitewindmills - XiShanYongYe-Chang diff --git a/pkg/controllers/binding/common.go b/pkg/controllers/binding/common.go index 0290156b7b24..d9eda632ec0c 100644 --- a/pkg/controllers/binding/common.go +++ b/pkg/controllers/binding/common.go @@ -17,6 +17,8 @@ limitations under the License. package binding import ( + "strconv" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -168,6 +170,9 @@ func mergeLabel(workload *unstructured.Unstructured, binding metav1.Object, scop func mergeAnnotations(workload *unstructured.Unstructured, binding metav1.Object, scope apiextensionsv1.ResourceScope) map[string]string { annotations := make(map[string]string) + if workload.GetGeneration() > 0 { + util.MergeAnnotation(workload, workv1alpha2.ResourceTemplateGenerationAnnotationKey, strconv.FormatInt(workload.GetGeneration(), 10)) + } if scope == apiextensionsv1.NamespaceScoped { util.MergeAnnotation(workload, workv1alpha2.ResourceBindingNamespaceAnnotationKey, binding.GetNamespace()) diff --git a/pkg/controllers/cronfederatedhpa/cronfederatedhpa_controller.go b/pkg/controllers/cronfederatedhpa/cronfederatedhpa_controller.go index 76833ec42eea..be26a564c85e 100755 --- a/pkg/controllers/cronfederatedhpa/cronfederatedhpa_controller.go +++ b/pkg/controllers/cronfederatedhpa/cronfederatedhpa_controller.go @@ -1,9 +1,12 @@ /* Copyright 2023 The Karmada Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/controllers/cronfederatedhpa/cronfederatedhpa_handler.go b/pkg/controllers/cronfederatedhpa/cronfederatedhpa_handler.go index 12a169680b18..a36ae8acbc1d 100755 --- a/pkg/controllers/cronfederatedhpa/cronfederatedhpa_handler.go +++ b/pkg/controllers/cronfederatedhpa/cronfederatedhpa_handler.go @@ -1,9 +1,12 @@ /* Copyright 2023 The Karmada Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/controllers/cronfederatedhpa/cronfederatedhpa_job.go b/pkg/controllers/cronfederatedhpa/cronfederatedhpa_job.go index c50b4541171f..e06db0fb9a01 100755 --- a/pkg/controllers/cronfederatedhpa/cronfederatedhpa_job.go +++ b/pkg/controllers/cronfederatedhpa/cronfederatedhpa_job.go @@ -1,5 +1,6 @@ /* Copyright 2023 The Karmada Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -32,6 +33,7 @@ import ( "k8s.io/client-go/util/retry" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" autoscalingv1alpha1 "github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1" "github.com/karmada-io/karmada/pkg/metrics" @@ -226,52 +228,55 @@ func (c *ScalingJob) addFailedExecutionHistory( cronFHPA *autoscalingv1alpha1.CronFederatedHPA, errMsg string) error { _, nextExecutionTime := c.scheduler.NextRun() - // Add success history record, return false if there is no such rule - addFailedHistoryFunc := func() bool { - exists := false - for index, rule := range cronFHPA.Status.ExecutionHistories { - if rule.RuleName != c.rule.Name { - continue - } - failedExecution := autoscalingv1alpha1.FailedExecution{ - ScheduleTime: rule.NextExecutionTime, - ExecutionTime: &metav1.Time{Time: time.Now()}, - Message: errMsg, - } - historyLimits := helper.GetCronFederatedHPAFailedHistoryLimits(c.rule) - if len(rule.FailedExecutions) > historyLimits-1 { - rule.FailedExecutions = rule.FailedExecutions[:historyLimits-1] - } - cronFHPA.Status.ExecutionHistories[index].FailedExecutions = - append([]autoscalingv1alpha1.FailedExecution{failedExecution}, rule.FailedExecutions...) - cronFHPA.Status.ExecutionHistories[index].NextExecutionTime = &metav1.Time{Time: nextExecutionTime} - exists = true - break + // Add failed history record + addFailedHistoryFunc := func(index int) { + failedExecution := autoscalingv1alpha1.FailedExecution{ + ScheduleTime: cronFHPA.Status.ExecutionHistories[index].NextExecutionTime, + ExecutionTime: &metav1.Time{Time: time.Now()}, + Message: errMsg, + } + historyLimits := helper.GetCronFederatedHPAFailedHistoryLimits(c.rule) + if len(cronFHPA.Status.ExecutionHistories[index].FailedExecutions) > historyLimits-1 { + cronFHPA.Status.ExecutionHistories[index].FailedExecutions = cronFHPA.Status.ExecutionHistories[index].FailedExecutions[:historyLimits-1] } + cronFHPA.Status.ExecutionHistories[index].FailedExecutions = + append([]autoscalingv1alpha1.FailedExecution{failedExecution}, cronFHPA.Status.ExecutionHistories[index].FailedExecutions...) + cronFHPA.Status.ExecutionHistories[index].NextExecutionTime = &metav1.Time{Time: nextExecutionTime} + } - return exists + index := c.findExecutionHistory(cronFHPA.Status.ExecutionHistories) + if index < 0 { + // The failed history does not exist, it means the rule deleted, so just ignore it. + return nil } - return retry.RetryOnConflict(retry.DefaultRetry, func() (err error) { - // If this history not exist, it means the rule is suspended or deleted, so just ignore it. - if exists := addFailedHistoryFunc(); !exists { + var operationResult controllerutil.OperationResult + if err := retry.RetryOnConflict(retry.DefaultRetry, func() (err error) { + operationResult, err = helper.UpdateStatus(context.Background(), c.client, cronFHPA, func() error { + addFailedHistoryFunc(index) return nil - } + }) + return err + }); err != nil { + klog.Errorf("Failed to add failed history record to CronFederatedHPA(%s/%s): %v", cronFHPA.Namespace, cronFHPA.Name, err) + return err + } - updateErr := c.client.Status().Update(context.Background(), cronFHPA) - if updateErr == nil { - klog.V(4).Infof("CronFederatedHPA(%s/%s) status has been updated successfully", cronFHPA.Namespace, cronFHPA.Name) - return nil - } + if operationResult == controllerutil.OperationResultUpdatedStatusOnly { + klog.V(4).Infof("CronFederatedHPA(%s/%s) status has been updated successfully", cronFHPA.Namespace, cronFHPA.Name) + } - updated := &autoscalingv1alpha1.CronFederatedHPA{} - if err = c.client.Get(context.Background(), client.ObjectKey{Namespace: cronFHPA.Namespace, Name: cronFHPA.Name}, updated); err == nil { - cronFHPA = updated - } else { - klog.Errorf("Get CronFederatedHPA(%s/%s) failed: %v", cronFHPA.Namespace, cronFHPA.Name, err) + return nil +} + +// findExecutionHistory finds the history record, returns -1 if there is no such rule. +func (c *ScalingJob) findExecutionHistory(histories []autoscalingv1alpha1.ExecutionHistory) int { + for index, rule := range histories { + if rule.RuleName == c.rule.Name { + return index } - return updateErr - }) + } + return -1 } func (c *ScalingJob) addSuccessExecutionHistory( @@ -279,52 +284,45 @@ func (c *ScalingJob) addSuccessExecutionHistory( appliedReplicas, appliedMinReplicas, appliedMaxReplicas *int32) error { _, nextExecutionTime := c.scheduler.NextRun() - // Add success history record, return false if there is no such rule - addSuccessHistoryFunc := func() bool { - exists := false - for index, rule := range cronFHPA.Status.ExecutionHistories { - if rule.RuleName != c.rule.Name { - continue - } - successExecution := autoscalingv1alpha1.SuccessfulExecution{ - ScheduleTime: rule.NextExecutionTime, - ExecutionTime: &metav1.Time{Time: time.Now()}, - AppliedReplicas: appliedReplicas, - AppliedMaxReplicas: appliedMaxReplicas, - AppliedMinReplicas: appliedMinReplicas, - } - historyLimits := helper.GetCronFederatedHPASuccessHistoryLimits(c.rule) - if len(rule.SuccessfulExecutions) > historyLimits-1 { - rule.SuccessfulExecutions = rule.SuccessfulExecutions[:historyLimits-1] - } - cronFHPA.Status.ExecutionHistories[index].SuccessfulExecutions = - append([]autoscalingv1alpha1.SuccessfulExecution{successExecution}, rule.SuccessfulExecutions...) - cronFHPA.Status.ExecutionHistories[index].NextExecutionTime = &metav1.Time{Time: nextExecutionTime} - exists = true - break + // Add success history record + addSuccessHistoryFunc := func(index int) { + successExecution := autoscalingv1alpha1.SuccessfulExecution{ + ScheduleTime: cronFHPA.Status.ExecutionHistories[index].NextExecutionTime, + ExecutionTime: &metav1.Time{Time: time.Now()}, + AppliedReplicas: appliedReplicas, + AppliedMaxReplicas: appliedMaxReplicas, + AppliedMinReplicas: appliedMinReplicas, + } + historyLimits := helper.GetCronFederatedHPASuccessHistoryLimits(c.rule) + if len(cronFHPA.Status.ExecutionHistories[index].SuccessfulExecutions) > historyLimits-1 { + cronFHPA.Status.ExecutionHistories[index].SuccessfulExecutions = cronFHPA.Status.ExecutionHistories[index].SuccessfulExecutions[:historyLimits-1] } + cronFHPA.Status.ExecutionHistories[index].SuccessfulExecutions = + append([]autoscalingv1alpha1.SuccessfulExecution{successExecution}, cronFHPA.Status.ExecutionHistories[index].SuccessfulExecutions...) + cronFHPA.Status.ExecutionHistories[index].NextExecutionTime = &metav1.Time{Time: nextExecutionTime} + } - return exists + index := c.findExecutionHistory(cronFHPA.Status.ExecutionHistories) + if index < 0 { + // The success history does not exist, it means the rule deleted, so just ignore it. + return nil } - return retry.RetryOnConflict(retry.DefaultRetry, func() (err error) { - // If this history not exist, it means the rule deleted, so just ignore it. - if exists := addSuccessHistoryFunc(); !exists { + var operationResult controllerutil.OperationResult + if err := retry.RetryOnConflict(retry.DefaultRetry, func() (err error) { + operationResult, err = helper.UpdateStatus(context.Background(), c.client, cronFHPA, func() error { + addSuccessHistoryFunc(index) return nil - } + }) + return err + }); err != nil { + klog.Errorf("Failed to add success history record to CronFederatedHPA(%s/%s): %v", cronFHPA.Namespace, cronFHPA.Name, err) + return err + } - updateErr := c.client.Status().Update(context.Background(), cronFHPA) - if updateErr == nil { - klog.V(4).Infof("CronFederatedHPA(%s/%s) status has been updated successfully", cronFHPA.Namespace, cronFHPA.Name) - return err - } + if operationResult == controllerutil.OperationResultUpdatedStatusOnly { + klog.V(4).Infof("CronFederatedHPA(%s/%s) status has been updated successfully", cronFHPA.Namespace, cronFHPA.Name) + } - updated := &autoscalingv1alpha1.CronFederatedHPA{} - if err = c.client.Get(context.Background(), client.ObjectKey{Namespace: cronFHPA.Namespace, Name: cronFHPA.Name}, updated); err == nil { - cronFHPA = updated - } else { - klog.Errorf("Get CronFederatedHPA(%s/%s) failed: %v", cronFHPA.Namespace, cronFHPA.Name, err) - } - return updateErr - }) + return nil } diff --git a/pkg/controllers/execution/execution_controller.go b/pkg/controllers/execution/execution_controller.go index d484ff90d4d8..8a398576c2bd 100644 --- a/pkg/controllers/execution/execution_controller.go +++ b/pkg/controllers/execution/execution_controller.go @@ -258,18 +258,11 @@ func (c *Controller) updateAppliedCondition(work *workv1alpha1.Work, status meta } return retry.RetryOnConflict(retry.DefaultRetry, func() (err error) { - meta.SetStatusCondition(&work.Status.Conditions, newWorkAppliedCondition) - updateErr := c.Status().Update(context.TODO(), work) - if updateErr == nil { + _, err = helper.UpdateStatus(context.Background(), c.Client, work, func() error { + meta.SetStatusCondition(&work.Status.Conditions, newWorkAppliedCondition) return nil - } - updated := &workv1alpha1.Work{} - if err = c.Get(context.TODO(), client.ObjectKey{Namespace: work.Namespace, Name: work.Name}, updated); err == nil { - work = updated - } else { - klog.Errorf("Failed to get the updated work(%s/%s), err: %v", work.Namespace, work.Name, err) - } - return updateErr + }) + return err }) } diff --git a/pkg/controllers/federatedhpa/metrics/client.go b/pkg/controllers/federatedhpa/metrics/client.go index 7e98b05a397b..6e63a6548394 100644 --- a/pkg/controllers/federatedhpa/metrics/client.go +++ b/pkg/controllers/federatedhpa/metrics/client.go @@ -72,24 +72,24 @@ func (c *resourceMetricsClient) GetResourceMetric(ctx context.Context, resource startTime := time.Now() defer metrics.ObserveFederatedHPAPullMetricsLatency(err, "ResourceMetric", startTime) - metrics, err := c.client.PodMetricses(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector.String()}) + podMetrics, err := c.client.PodMetricses(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector.String()}) if err != nil { return nil, time.Time{}, fmt.Errorf("unable to fetch metrics from resource metrics API: %v", err) } - if len(metrics.Items) == 0 { + if len(podMetrics.Items) == 0 { return nil, time.Time{}, fmt.Errorf("no metrics returned from resource metrics API") } var res PodMetricsInfo if container != "" { - res, err = getContainerMetrics(metrics.Items, resource, container) + res, err = getContainerMetrics(podMetrics.Items, resource, container) if err != nil { return nil, time.Time{}, fmt.Errorf("failed to get container metrics: %v", err) } } else { - res = getPodMetrics(metrics.Items, resource) + res = getPodMetrics(podMetrics.Items, resource) } - timestamp := metrics.Items[0].Timestamp.Time + timestamp := podMetrics.Items[0].Timestamp.Time return res, timestamp, nil } @@ -156,17 +156,17 @@ func (c *customMetricsClient) GetRawMetric(metricName string, namespace string, startTime := time.Now() defer metrics.ObserveFederatedHPAPullMetricsLatency(err, "RawMetric", startTime) - metrics, err := c.client.NamespacedMetrics(namespace).GetForObjects(schema.GroupKind{Kind: "Pod"}, selector, metricName, metricSelector) + metricList, err := c.client.NamespacedMetrics(namespace).GetForObjects(schema.GroupKind{Kind: "Pod"}, selector, metricName, metricSelector) if err != nil { return nil, time.Time{}, fmt.Errorf("unable to fetch metrics from custom metrics API: %v", err) } - if len(metrics.Items) == 0 { + if len(metricList.Items) == 0 { return nil, time.Time{}, fmt.Errorf("no metrics returned from custom metrics API") } - res := make(PodMetricsInfo, len(metrics.Items)) - for _, m := range metrics.Items { + res := make(PodMetricsInfo, len(metricList.Items)) + for _, m := range metricList.Items { window := metricServerDefaultMetricWindow if m.WindowSeconds != nil { window = time.Duration(*m.WindowSeconds) * time.Second @@ -174,13 +174,13 @@ func (c *customMetricsClient) GetRawMetric(metricName string, namespace string, res[m.DescribedObject.Name] = PodMetric{ Timestamp: m.Timestamp.Time, Window: window, - Value: int64(m.Value.MilliValue()), + Value: m.Value.MilliValue(), } m.Value.MilliValue() } - timestamp := metrics.Items[0].Timestamp.Time + timestamp := metricList.Items[0].Timestamp.Time return res, timestamp, nil } @@ -225,19 +225,19 @@ func (c *externalMetricsClient) GetExternalMetric(metricName, namespace string, startTime := time.Now() defer metrics.ObserveFederatedHPAPullMetricsLatency(err, "ExternalMetric", startTime) - metrics, err := c.client.NamespacedMetrics(namespace).List(metricName, selector) + externalMetrics, err := c.client.NamespacedMetrics(namespace).List(metricName, selector) if err != nil { return []int64{}, time.Time{}, fmt.Errorf("unable to fetch metrics from external metrics API: %v", err) } - if len(metrics.Items) == 0 { + if len(externalMetrics.Items) == 0 { return nil, time.Time{}, fmt.Errorf("no metrics returned from external metrics API") } res := make([]int64, 0) - for _, m := range metrics.Items { + for _, m := range externalMetrics.Items { res = append(res, m.Value.MilliValue()) } - timestamp := metrics.Items[0].Timestamp.Time + timestamp := externalMetrics.Items[0].Timestamp.Time return res, timestamp, nil } diff --git a/pkg/controllers/federatedresourcequota/federated_resource_quota_status_controller.go b/pkg/controllers/federatedresourcequota/federated_resource_quota_status_controller.go index b7d1533022c6..23039535f0bd 100644 --- a/pkg/controllers/federatedresourcequota/federated_resource_quota_status_controller.go +++ b/pkg/controllers/federatedresourcequota/federated_resource_quota_status_controller.go @@ -160,20 +160,11 @@ func (c *StatusController) collectQuotaStatus(quota *policyv1alpha1.FederatedRes } return retry.RetryOnConflict(retry.DefaultRetry, func() error { - quota.Status = *quotaStatus - updateErr := c.Status().Update(context.TODO(), quota) - if updateErr == nil { + _, err = helper.UpdateStatus(context.Background(), c.Client, quota, func() error { + quota.Status = *quotaStatus return nil - } - - updated := &policyv1alpha1.FederatedResourceQuota{} - if err = c.Get(context.TODO(), client.ObjectKey{Namespace: quota.Namespace, Name: quota.Name}, updated); err == nil { - quota = updated - } else { - klog.Errorf("Failed to get updated federatedResourceQuota(%s): %v", klog.KObj(quota).String(), err) - } - - return updateErr + }) + return err }) } diff --git a/pkg/controllers/mcs/endpointslice_controller.go b/pkg/controllers/mcs/endpointslice_controller.go index e72cbbe50b41..d63ce47e7c3c 100644 --- a/pkg/controllers/mcs/endpointslice_controller.go +++ b/pkg/controllers/mcs/endpointslice_controller.go @@ -24,7 +24,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" - utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/tools/record" "k8s.io/klog/v2" controllerruntime "sigs.k8s.io/controller-runtime" @@ -57,24 +56,7 @@ func (c *EndpointSliceController) Reconcile(ctx context.Context, req controllerr work := &workv1alpha1.Work{} if err := c.Client.Get(ctx, req.NamespacedName, work); err != nil { if apierrors.IsNotFound(err) { - // Clean up derived EndpointSlices after work has been removed. - endpointSlices := &discoveryv1.EndpointSliceList{} - if err = c.List(context.TODO(), endpointSlices, client.HasLabels{workv1alpha2.WorkPermanentIDLabel}); err != nil { - return controllerruntime.Result{}, err - } - - var errs []error - for i, es := range endpointSlices.Items { - if es.Annotations[workv1alpha2.WorkNamespaceAnnotation] == req.Namespace && - es.Annotations[workv1alpha2.WorkNameAnnotation] == req.Name { - if err := c.Delete(context.TODO(), &endpointSlices.Items[i]); err != nil && !apierrors.IsNotFound(err) { - klog.Errorf("Failed to delete endpointslice(%s/%s) after the work(%s/%s) has been removed, err: %v", - es.Namespace, es.Name, req.Namespace, req.Name, err) - errs = append(errs, err) - } - } - } - return controllerruntime.Result{}, utilerrors.NewAggregate(errs) + return controllerruntime.Result{}, nil } return controllerruntime.Result{}, err } @@ -161,7 +143,10 @@ func (c *EndpointSliceController) collectEndpointSliceFromWork(work *workv1alpha desiredEndpointSlice.Labels = util.DedupeAndMergeLabels(desiredEndpointSlice.Labels, map[string]string{ workv1alpha2.WorkPermanentIDLabel: work.Labels[workv1alpha2.WorkPermanentIDLabel], discoveryv1.LabelServiceName: names.GenerateDerivedServiceName(work.Labels[util.ServiceNameLabel]), - util.ManagedByKarmadaLabel: util.ManagedByKarmadaLabelValue, + }) + desiredEndpointSlice.Annotations = util.DedupeAndMergeAnnotations(desiredEndpointSlice.Annotations, map[string]string{ + workv1alpha2.WorkNamespaceAnnotation: work.Namespace, + workv1alpha2.WorkNameAnnotation: work.Name, }) if err = helper.CreateOrUpdateEndpointSlice(c.Client, desiredEndpointSlice); err != nil { diff --git a/pkg/controllers/mcs/service_import_controller.go b/pkg/controllers/mcs/service_import_controller.go index 329e5256e0c5..cde4cbe41e39 100644 --- a/pkg/controllers/mcs/service_import_controller.go +++ b/pkg/controllers/mcs/service_import_controller.go @@ -31,7 +31,7 @@ import ( mcsv1alpha1 "sigs.k8s.io/mcs-api/pkg/apis/v1alpha1" "github.com/karmada-io/karmada/pkg/events" - "github.com/karmada-io/karmada/pkg/util" + "github.com/karmada-io/karmada/pkg/util/helper" "github.com/karmada-io/karmada/pkg/util/names" ) @@ -103,9 +103,6 @@ func (c *ServiceImportController) deriveServiceFromServiceImport(svcImport *mcsv ObjectMeta: metav1.ObjectMeta{ Namespace: svcImport.Namespace, Name: names.GenerateDerivedServiceName(svcImport.Name), - Labels: map[string]string{ - util.ManagedByKarmadaLabel: util.ManagedByKarmadaLabelValue, - }, }, Spec: corev1.ServiceSpec{ Type: corev1.ServiceTypeClusterIP, @@ -153,24 +150,15 @@ func (c *ServiceImportController) updateServiceStatus(svcImport *mcsv1alpha1.Ser } err := retry.RetryOnConflict(retry.DefaultRetry, func() (err error) { - derivedService.Status = corev1.ServiceStatus{ - LoadBalancer: corev1.LoadBalancerStatus{ - Ingress: ingress, - }, - } - updateErr := c.Status().Update(context.TODO(), derivedService) - if updateErr == nil { + _, err = helper.UpdateStatus(context.Background(), c.Client, derivedService, func() error { + derivedService.Status = corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{ + Ingress: ingress, + }, + } return nil - } - - updated := &corev1.Service{} - if err = c.Get(context.TODO(), client.ObjectKey{Namespace: derivedService.Namespace, Name: derivedService.Name}, updated); err == nil { - derivedService = updated - } else { - klog.Errorf("Failed to get updated service %s/%s: %v", derivedService.Namespace, derivedService.Name, err) - } - - return updateErr + }) + return err }) if err != nil { diff --git a/pkg/controllers/multiclusterservice/endpointslice_collect_controller.go b/pkg/controllers/multiclusterservice/endpointslice_collect_controller.go index 08cd0292b4f6..0952c7dbc160 100644 --- a/pkg/controllers/multiclusterservice/endpointslice_collect_controller.go +++ b/pkg/controllers/multiclusterservice/endpointslice_collect_controller.go @@ -402,7 +402,7 @@ func getEndpointSliceWorkMeta(c client.Client, ns string, workName string, endpo return metav1.ObjectMeta{}, err } - labels := map[string]string{ + ls := map[string]string{ util.MultiClusterServiceNamespaceLabel: endpointSlice.GetNamespace(), util.MultiClusterServiceNameLabel: endpointSlice.GetLabels()[discoveryv1.LabelServiceName], // indicate the Work should be not propagated since it's collected resource. @@ -410,18 +410,18 @@ func getEndpointSliceWorkMeta(c client.Client, ns string, workName string, endpo util.EndpointSliceWorkManagedByLabel: util.MultiClusterServiceKind, } if existWork.Labels == nil || (err != nil && apierrors.IsNotFound(err)) { - workMeta := metav1.ObjectMeta{Name: workName, Namespace: ns, Labels: labels} + workMeta := metav1.ObjectMeta{Name: workName, Namespace: ns, Labels: ls} return workMeta, nil } - labels = util.DedupeAndMergeLabels(labels, existWork.Labels) + ls = util.DedupeAndMergeLabels(ls, existWork.Labels) if value, ok := existWork.Labels[util.EndpointSliceWorkManagedByLabel]; ok { controllerSet := sets.New[string]() controllerSet.Insert(strings.Split(value, ".")...) controllerSet.Insert(util.MultiClusterServiceKind) - labels[util.EndpointSliceWorkManagedByLabel] = strings.Join(controllerSet.UnsortedList(), ".") + ls[util.EndpointSliceWorkManagedByLabel] = strings.Join(controllerSet.UnsortedList(), ".") } - workMeta := metav1.ObjectMeta{Name: workName, Namespace: ns, Labels: labels} + workMeta := metav1.ObjectMeta{Name: workName, Namespace: ns, Labels: ls} return workMeta, nil } diff --git a/pkg/controllers/multiclusterservice/endpointslice_dispatch_controller.go b/pkg/controllers/multiclusterservice/endpointslice_dispatch_controller.go index 8175a65ef43f..b5d7e4261c5c 100644 --- a/pkg/controllers/multiclusterservice/endpointslice_dispatch_controller.go +++ b/pkg/controllers/multiclusterservice/endpointslice_dispatch_controller.go @@ -127,18 +127,11 @@ func (c *EndpointsliceDispatchController) updateEndpointSliceDispatched(mcs *net } return retry.RetryOnConflict(retry.DefaultRetry, func() (err error) { - meta.SetStatusCondition(&mcs.Status.Conditions, EndpointSliceCollected) - updateErr := c.Status().Update(context.TODO(), mcs) - if updateErr == nil { + _, err = helper.UpdateStatus(context.Background(), c.Client, mcs, func() error { + meta.SetStatusCondition(&mcs.Status.Conditions, EndpointSliceCollected) return nil - } - updated := &networkingv1alpha1.MultiClusterService{} - if err = c.Get(context.TODO(), client.ObjectKey{Namespace: mcs.Namespace, Name: mcs.Name}, updated); err == nil { - mcs = updated - } else { - klog.Errorf("Failed to get updated MultiClusterService %s/%s: %v", mcs.Namespace, mcs.Name, err) - } - return updateErr + }) + return err }) } diff --git a/pkg/controllers/multiclusterservice/mcs_controller.go b/pkg/controllers/multiclusterservice/mcs_controller.go index 3d2e511c17fa..bf17f11b4a56 100644 --- a/pkg/controllers/multiclusterservice/mcs_controller.go +++ b/pkg/controllers/multiclusterservice/mcs_controller.go @@ -521,18 +521,11 @@ func (c *MCSController) updateMultiClusterServiceStatus(mcs *networkingv1alpha1. } return retry.RetryOnConflict(retry.DefaultRetry, func() (err error) { - meta.SetStatusCondition(&mcs.Status.Conditions, serviceAppliedCondition) - updateErr := c.Status().Update(context.TODO(), mcs) - if updateErr == nil { + _, err = helper.UpdateStatus(context.Background(), c.Client, mcs, func() error { + meta.SetStatusCondition(&mcs.Status.Conditions, serviceAppliedCondition) return nil - } - updated := &networkingv1alpha1.MultiClusterService{} - if err = c.Get(context.TODO(), client.ObjectKey{Namespace: mcs.Namespace, Name: mcs.Name}, updated); err == nil { - mcs = updated - } else { - klog.Errorf("Failed to get updated MultiClusterService %s/%s: %v", mcs.Namespace, mcs.Name, err) - } - return updateErr + }) + return err }) } diff --git a/pkg/controllers/remediation/remedy_controller.go b/pkg/controllers/remediation/remedy_controller.go index c8424dea1a56..19688765c4a3 100644 --- a/pkg/controllers/remediation/remedy_controller.go +++ b/pkg/controllers/remediation/remedy_controller.go @@ -18,10 +18,8 @@ package remediation import ( "context" - "reflect" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" "k8s.io/klog/v2" controllerruntime "sigs.k8s.io/controller-runtime" @@ -33,6 +31,7 @@ import ( clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1" remedyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1" "github.com/karmada-io/karmada/pkg/sharedcli/ratelimiterflag" + "github.com/karmada-io/karmada/pkg/util/helper" ) // ControllerName is the controller name that will be used when reporting events. @@ -70,30 +69,17 @@ func (c *RemedyController) Reconcile(ctx context.Context, req controllerruntime. } actions := calculateActions(clusterRelatedRemedies, cluster) - err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - if reflect.DeepEqual(actions, cluster.Status.RemedyActions) { + if err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + _, err = helper.UpdateStatus(ctx, c.Client, cluster, func() error { + cluster.Status.RemedyActions = actions return nil - } - cluster.Status.RemedyActions = actions - updateErr := c.Client.Status().Update(ctx, cluster) - if updateErr == nil { - return nil - } - - updatedCluster := &clusterv1alpha1.Cluster{} - err = c.Client.Get(ctx, types.NamespacedName{Name: cluster.Name}, updatedCluster) - if err == nil { - cluster = updatedCluster - } else { - klog.Errorf("Failed to get updated cluster(%s): %v", cluster.Name, err) - } - return updateErr - }) - if err != nil { + }) + return err + }); err != nil { klog.Errorf("Failed to sync cluster(%s) remedy actions: %v", cluster.Name, err) return controllerruntime.Result{}, err } - klog.V(4).Infof("Success to sync cluster(%s) remedy actions", cluster.Name) + klog.V(4).Infof("Success to sync cluster(%s) remedy actions: %v", cluster.Name, actions) return controllerruntime.Result{}, nil } diff --git a/pkg/controllers/status/cluster_status_controller.go b/pkg/controllers/status/cluster_status_controller.go index d94bff005344..44afe1ee1009 100644 --- a/pkg/controllers/status/cluster_status_controller.go +++ b/pkg/controllers/status/cluster_status_controller.go @@ -277,19 +277,15 @@ func (c *ClusterStatusController) updateStatusIfNeeded(cluster *clusterv1alpha1. if !equality.Semantic.DeepEqual(cluster.Status, currentClusterStatus) { klog.V(4).Infof("Start to update cluster status: %s", cluster.Name) err := retry.RetryOnConflict(retry.DefaultRetry, func() (err error) { - cluster.Status = currentClusterStatus - updateErr := c.Status().Update(context.TODO(), cluster) - if updateErr == nil { + _, err = helper.UpdateStatus(context.Background(), c.Client, cluster, func() error { + cluster.Status.KubernetesVersion = currentClusterStatus.KubernetesVersion + cluster.Status.APIEnablements = currentClusterStatus.APIEnablements + cluster.Status.Conditions = currentClusterStatus.Conditions + cluster.Status.NodeSummary = currentClusterStatus.NodeSummary + cluster.Status.ResourceSummary = currentClusterStatus.ResourceSummary return nil - } - - updated := &clusterv1alpha1.Cluster{} - if err = c.Get(context.TODO(), client.ObjectKey{Namespace: cluster.Namespace, Name: cluster.Name}, updated); err == nil { - cluster = updated - } else { - klog.Errorf("Failed to get updated cluster %s: %v", cluster.Name, err) - } - return updateErr + }) + return err }) if err != nil { klog.Errorf("Failed to update health status of the member cluster: %v, err is : %v", cluster.Name, err) diff --git a/pkg/controllers/status/work_status_controller.go b/pkg/controllers/status/work_status_controller.go index 7f71c5873cd2..7178b00ab5e6 100644 --- a/pkg/controllers/status/work_status_controller.go +++ b/pkg/controllers/status/work_status_controller.go @@ -326,22 +326,11 @@ func (c *WorkStatusController) updateAppliedCondition(work *workv1alpha1.Work, s } err := retry.RetryOnConflict(retry.DefaultRetry, func() (err error) { - workStatus := work.Status.DeepCopy() - meta.SetStatusCondition(&work.Status.Conditions, newWorkAppliedCondition) - if reflect.DeepEqual(*workStatus, work.Status) { + _, err = helper.UpdateStatus(context.Background(), c.Client, work, func() error { + meta.SetStatusCondition(&work.Status.Conditions, newWorkAppliedCondition) return nil - } - updateErr := c.Status().Update(context.TODO(), work) - if updateErr == nil { - return nil - } - updated := &workv1alpha1.Work{} - if err = c.Get(context.TODO(), client.ObjectKey{Namespace: work.Namespace, Name: work.Name}, updated); err == nil { - work = updated - } else { - klog.Errorf("Failed to get updated work %s/%s: %s", work.Namespace, work.Name, err.Error()) - } - return updateErr + }) + return err }) if err != nil { @@ -386,25 +375,12 @@ func (c *WorkStatusController) reflectStatus(work *workv1alpha1.Work, clusterObj Health: resourceHealth, } - workCopy := work.DeepCopy() return retry.RetryOnConflict(retry.DefaultRetry, func() (err error) { - manifestStatuses := c.mergeStatus(workCopy.Status.ManifestStatuses, manifestStatus) - if reflect.DeepEqual(workCopy.Status.ManifestStatuses, manifestStatuses) { + _, err = helper.UpdateStatus(context.Background(), c.Client, work, func() error { + work.Status.ManifestStatuses = c.mergeStatus(work.Status.ManifestStatuses, manifestStatus) return nil - } - workCopy.Status.ManifestStatuses = manifestStatuses - updateErr := c.Status().Update(context.TODO(), workCopy) - if updateErr == nil { - return nil - } - - updated := &workv1alpha1.Work{} - if err = c.Get(context.TODO(), client.ObjectKey{Namespace: workCopy.Namespace, Name: workCopy.Name}, updated); err == nil { - workCopy = updated - } else { - klog.Errorf("Failed to get updated work %s/%s: %v", workCopy.Namespace, workCopy.Name, err) - } - return updateErr + }) + return err }) } diff --git a/pkg/descheduler/descheduler.go b/pkg/descheduler/descheduler.go index f5689672fa81..f37d79db5758 100644 --- a/pkg/descheduler/descheduler.go +++ b/pkg/descheduler/descheduler.go @@ -45,6 +45,7 @@ import ( "github.com/karmada-io/karmada/pkg/util" "github.com/karmada-io/karmada/pkg/util/fedinformer" "github.com/karmada-io/karmada/pkg/util/gclient" + "github.com/karmada-io/karmada/pkg/util/grpcconnection" ) const ( @@ -65,7 +66,7 @@ type Descheduler struct { schedulerEstimatorCache *estimatorclient.SchedulerEstimatorCache schedulerEstimatorServicePrefix string - schedulerEstimatorPort int + schedulerEstimatorClientConfig *grpcconnection.ClientConfig schedulerEstimatorWorker util.AsyncWorker unschedulableThreshold time.Duration @@ -77,15 +78,21 @@ type Descheduler struct { func NewDescheduler(karmadaClient karmadaclientset.Interface, kubeClient kubernetes.Interface, opts *options.Options) *Descheduler { factory := informerfactory.NewSharedInformerFactory(karmadaClient, 0) desched := &Descheduler{ - KarmadaClient: karmadaClient, - KubeClient: kubeClient, - informerFactory: factory, - bindingInformer: factory.Work().V1alpha2().ResourceBindings().Informer(), - bindingLister: factory.Work().V1alpha2().ResourceBindings().Lister(), - clusterInformer: factory.Cluster().V1alpha1().Clusters().Informer(), - clusterLister: factory.Cluster().V1alpha1().Clusters().Lister(), - schedulerEstimatorCache: estimatorclient.NewSchedulerEstimatorCache(), - schedulerEstimatorPort: opts.SchedulerEstimatorPort, + KarmadaClient: karmadaClient, + KubeClient: kubeClient, + informerFactory: factory, + bindingInformer: factory.Work().V1alpha2().ResourceBindings().Informer(), + bindingLister: factory.Work().V1alpha2().ResourceBindings().Lister(), + clusterInformer: factory.Cluster().V1alpha1().Clusters().Informer(), + clusterLister: factory.Cluster().V1alpha1().Clusters().Lister(), + schedulerEstimatorCache: estimatorclient.NewSchedulerEstimatorCache(), + schedulerEstimatorClientConfig: &grpcconnection.ClientConfig{ + InsecureSkipServerVerify: opts.InsecureSkipEstimatorVerify, + ServerAuthCAFile: opts.SchedulerEstimatorCaFile, + CertFile: opts.SchedulerEstimatorCertFile, + KeyFile: opts.SchedulerEstimatorKeyFile, + TargetPort: opts.SchedulerEstimatorPort, + }, schedulerEstimatorServicePrefix: opts.SchedulerEstimatorServicePrefix, unschedulableThreshold: opts.UnschedulableThreshold.Duration, deschedulingInterval: opts.DeschedulingInterval.Duration, @@ -273,7 +280,7 @@ func (d *Descheduler) establishEstimatorConnections() { return } for i := range clusterList.Items { - if err = estimatorclient.EstablishConnection(d.KubeClient, clusterList.Items[i].Name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorPort); err != nil { + if err = estimatorclient.EstablishConnection(d.KubeClient, clusterList.Items[i].Name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorClientConfig); err != nil { klog.Error(err) } } @@ -293,7 +300,7 @@ func (d *Descheduler) reconcileEstimatorConnection(key util.QueueKey) error { } return err } - return estimatorclient.EstablishConnection(d.KubeClient, name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorPort) + return estimatorclient.EstablishConnection(d.KubeClient, name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorClientConfig) } func (d *Descheduler) recordDescheduleResultEventForResourceBinding(rb *workv1alpha2.ResourceBinding, message string, err error) { diff --git a/pkg/estimator/client/cache.go b/pkg/estimator/client/cache.go index 3c278e240f0d..cc7ad0f6a9c9 100644 --- a/pkg/estimator/client/cache.go +++ b/pkg/estimator/client/cache.go @@ -27,6 +27,7 @@ import ( estimatorservice "github.com/karmada-io/karmada/pkg/estimator/service" "github.com/karmada-io/karmada/pkg/util" + "github.com/karmada-io/karmada/pkg/util/grpcconnection" "github.com/karmada-io/karmada/pkg/util/names" ) @@ -96,19 +97,19 @@ func (c *SchedulerEstimatorCache) GetClient(name string) (estimatorservice.Estim } // EstablishConnection establishes a new gRPC connection with the specified cluster scheduler estimator. -func EstablishConnection(kubeClient kubernetes.Interface, name string, estimatorCache *SchedulerEstimatorCache, estimatorServicePrefix string, port int) error { +func EstablishConnection(kubeClient kubernetes.Interface, name string, estimatorCache *SchedulerEstimatorCache, estimatorServicePrefix string, grpcConfig *grpcconnection.ClientConfig) error { if estimatorCache.IsEstimatorExist(name) { return nil } serverAddr, err := resolveCluster(kubeClient, util.NamespaceKarmadaSystem, - names.GenerateEstimatorServiceName(estimatorServicePrefix, name), int32(port)) + names.GenerateEstimatorServiceName(estimatorServicePrefix, name), int32(grpcConfig.TargetPort)) if err != nil { return err } klog.Infof("Start dialing estimator server(%s) of cluster(%s).", serverAddr, name) - cc, err := util.Dial(serverAddr, 5*time.Second) + cc, err := grpcConfig.DialWithTimeOut(serverAddr, 5*time.Second) if err != nil { klog.Errorf("Failed to dial cluster(%s): %v.", name, err) return err diff --git a/pkg/estimator/client/service.go b/pkg/estimator/client/service.go index 323cd7ca8661..c1216557763c 100644 --- a/pkg/estimator/client/service.go +++ b/pkg/estimator/client/service.go @@ -41,7 +41,7 @@ func resolveCluster(kubeClient kubernetes.Interface, namespace, id string, port * But the Service resource is defined in Host Kubernetes Cluster. So we cannot get its content here. * The best thing we can do is just glue host:port together, and try to connect to it. */ - return net.JoinHostPort(id, fmt.Sprintf("%d", port)), nil + return net.JoinHostPort(fmt.Sprintf("%s.%s.svc.cluster.local", id, namespace), fmt.Sprintf("%d", port)), nil } return "", err diff --git a/pkg/estimator/server/server.go b/pkg/estimator/server/server.go index d8b6156c7f26..693f435b9613 100644 --- a/pkg/estimator/server/server.go +++ b/pkg/estimator/server/server.go @@ -23,7 +23,6 @@ import ( "time" "github.com/kr/pretty" - "google.golang.org/grpc" "google.golang.org/grpc/metadata" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -53,6 +52,7 @@ import ( "github.com/karmada-io/karmada/pkg/util/fedinformer" "github.com/karmada-io/karmada/pkg/util/fedinformer/genericmanager" "github.com/karmada-io/karmada/pkg/util/fedinformer/keys" + "github.com/karmada-io/karmada/pkg/util/grpcconnection" "github.com/karmada-io/karmada/pkg/util/helper" schedcache "github.com/karmada-io/karmada/pkg/util/lifted/scheduler/cache" "github.com/karmada-io/karmada/pkg/util/lifted/scheduler/framework/parallelize" @@ -73,7 +73,6 @@ var ( // AccurateSchedulerEstimatorServer is the gRPC server of a cluster accurate scheduler estimator. // Please see https://github.com/karmada-io/karmada/pull/580 (#580). type AccurateSchedulerEstimatorServer struct { - port int clusterName string kubeClient kubernetes.Interface restMapper meta.RESTMapper @@ -85,6 +84,8 @@ type AccurateSchedulerEstimatorServer struct { estimateFramework framework.Framework Cache schedcache.Cache + + GrpcConfig *grpcconnection.ServerConfig } // NewEstimatorServer creates an instance of AccurateSchedulerEstimatorServer. @@ -101,7 +102,6 @@ func NewEstimatorServer( informerFactory.InformerFor(&corev1.Pod{}, newPodInformer) es := &AccurateSchedulerEstimatorServer{ - port: opts.ServerPort, clusterName: opts.ClusterName, kubeClient: kubeClient, restMapper: restMapper, @@ -113,6 +113,13 @@ func NewEstimatorServer( }, parallelizer: parallelize.NewParallelizer(opts.Parallelism), Cache: schedcache.New(durationToExpireAssumedPod, stopChan), + GrpcConfig: &grpcconnection.ServerConfig{ + InsecureSkipClientVerify: opts.InsecureSkipGrpcClientVerify, + ClientAuthCAFile: opts.GrpcClientCaFile, + CertFile: opts.GrpcAuthCertFile, + KeyFile: opts.GrpcAuthKeyFile, + ServerPort: opts.ServerPort, + }, } // ignore the error here because the informers haven't been started _ = informerFactory.Core().V1().Nodes().Informer().SetTransform(fedinformer.StripUnusedFields) @@ -154,14 +161,17 @@ func (es *AccurateSchedulerEstimatorServer) Start(ctx context.Context) error { } // Listen a port and register the gRPC server. - l, err := net.Listen("tcp", fmt.Sprintf(":%d", es.port)) + l, err := net.Listen("tcp", fmt.Sprintf(":%d", es.GrpcConfig.ServerPort)) if err != nil { - return fmt.Errorf("failed to listen port %d: %v", es.port, err) + return fmt.Errorf("failed to listen port %d: %v", es.GrpcConfig.ServerPort, err) } - klog.Infof("Listening port: %d", es.port) + klog.Infof("Listening port: %d", es.GrpcConfig.ServerPort) defer l.Close() - s := grpc.NewServer() + s, err := es.GrpcConfig.NewServer() + if err != nil { + return fmt.Errorf("failed to create grpc server: %v", err) + } estimatorservice.RegisterEstimatorServer(s, es) // Graceful stop when the context is cancelled. diff --git a/pkg/karmadactl/addons/descheduler/manifests.go b/pkg/karmadactl/addons/descheduler/manifests.go index af1bd357c56b..711b1ebbc025 100644 --- a/pkg/karmadactl/addons/descheduler/manifests.go +++ b/pkg/karmadactl/addons/descheduler/manifests.go @@ -47,6 +47,9 @@ spec: - --kubeconfig=/etc/kubeconfig - --bind-address=0.0.0.0 - --leader-elect-resource-namespace={{ .Namespace }} + - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt + - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt + - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key - --v=4 livenessProbe: httpGet: @@ -58,10 +61,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: k8s-certs + mountPath: /etc/karmada/pki + readOnly: true - name: kubeconfig subPath: kubeconfig mountPath: /etc/kubeconfig volumes: + - name: k8s-certs + secret: + secretName: karmada-cert - name: kubeconfig secret: secretName: kubeconfig diff --git a/pkg/karmadactl/addons/estimator/manifests.go b/pkg/karmadactl/addons/estimator/manifests.go index a11e29212f99..3eb58a9d977e 100644 --- a/pkg/karmadactl/addons/estimator/manifests.go +++ b/pkg/karmadactl/addons/estimator/manifests.go @@ -48,6 +48,10 @@ spec: - /bin/karmada-scheduler-estimator - --kubeconfig=/etc/{{ .MemberClusterName}}-kubeconfig - --cluster-name={{ .MemberClusterName}} + - --grpc-auth-cert-file=/etc/karmada/pki/karmada.crt + - --grpc-auth-key-file=/etc/karmada/pki/karmada.key + - --client-cert-auth=true + - --grpc-client-ca-file=/etc/karmada/pki/ca.crt livenessProbe: httpGet: path: /healthz @@ -58,10 +62,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: k8s-certs + mountPath: /etc/karmada/pki + readOnly: true - name: member-kubeconfig subPath: {{ .MemberClusterName}}-kubeconfig mountPath: /etc/{{ .MemberClusterName}}-kubeconfig volumes: + - name: k8s-certs + secret: + secretName: karmada-cert - name: member-kubeconfig secret: secretName: {{ .MemberClusterName}}-kubeconfig diff --git a/pkg/karmadactl/cmdinit/cmdinit.go b/pkg/karmadactl/cmdinit/cmdinit.go index fc67a5ffe373..ab946f4ec365 100644 --- a/pkg/karmadactl/cmdinit/cmdinit.go +++ b/pkg/karmadactl/cmdinit/cmdinit.go @@ -120,7 +120,7 @@ func NewCmdInit(parentCommand string) *cobra.Command { // kube image registry flags.StringVarP(&opts.KubeImageMirrorCountry, "kube-image-mirror-country", "", "", "Country code of the kube image registry to be used. For Chinese mainland users, set it to cn") flags.StringVarP(&opts.KubeImageRegistry, "kube-image-registry", "", "", "Kube image registry. For Chinese mainland users, you may use local gcr.io mirrors such as registry.cn-hangzhou.aliyuncs.com/google_containers to override default kube image registry") - flags.StringVar(&opts.KubeImageTag, "kube-image-tag", "v1.27.11", "Choose a specific Kubernetes version for the control plane.") + flags.StringVar(&opts.KubeImageTag, "kube-image-tag", "v1.28.9", "Choose a specific Kubernetes version for the control plane.") // cert flags.StringVar(&opts.ExternalIP, "cert-external-ip", "", "the external IP of Karmada certificate (e.g 192.168.1.2,172.16.1.2)") flags.StringVar(&opts.ExternalDNS, "cert-external-dns", "", "the external DNS of Karmada certificate (e.g localhost,localhost.com)") diff --git a/pkg/karmadactl/cmdinit/kubernetes/deployments.go b/pkg/karmadactl/cmdinit/kubernetes/deployments.go index bb0ea0479e69..f7bacf7522f1 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/deployments.go +++ b/pkg/karmadactl/cmdinit/kubernetes/deployments.go @@ -454,6 +454,9 @@ func (i *CommandInitOption) makeKarmadaSchedulerDeployment() *appsv1.Deployment "--secure-port=10351", "--enable-scheduler-estimator=true", "--leader-elect=true", + "--scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt", + "--scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt", + "--scheduler-estimator-key-file=/etc/karmada/pki/karmada.key", fmt.Sprintf("--leader-elect-resource-namespace=%s", i.Namespace), "--v=4", }, @@ -465,6 +468,11 @@ func (i *CommandInitOption) makeKarmadaSchedulerDeployment() *appsv1.Deployment MountPath: kubeConfigContainerMountPath, SubPath: KubeConfigSecretAndMountName, }, + { + Name: globaloptions.KarmadaCertsName, + ReadOnly: true, + MountPath: karmadaCertsVolumeMountPath, + }, }, }, }, @@ -477,6 +485,14 @@ func (i *CommandInitOption) makeKarmadaSchedulerDeployment() *appsv1.Deployment }, }, }, + { + Name: globaloptions.KarmadaCertsName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: globaloptions.KarmadaCertsName, + }, + }, + }, }, Tolerations: []corev1.Toleration{ { diff --git a/pkg/karmadactl/join/join.go b/pkg/karmadactl/join/join.go index 9ed409101ba4..fd773025f4f1 100644 --- a/pkg/karmadactl/join/join.go +++ b/pkg/karmadactl/join/join.go @@ -107,6 +107,15 @@ type CommandJoinOption struct { // more details about running Kubernetes in multiple zones. ClusterZones []string + // KarmadaAs represents the username to impersonate for the operation in karmada control plane. User could be a regular user or a service account in a namespace + KarmadaAs string + + // KarmadaAsGroups represents groups to impersonate for the operation in karmada control plane, this flag can be repeated to specify multiple groups + KarmadaAsGroups []string + + // KarmadaAsUID represents the UID to impersonate for the operation in karmada control plane. + KarmadaAsUID string + // DryRun tells if run the command in dry-run mode, without making any server requests. DryRun bool } @@ -150,6 +159,12 @@ func (j *CommandJoinOption) AddFlags(flags *pflag.FlagSet) { flags.StringVar(&j.ClusterProvider, "cluster-provider", "", "Provider of the joining cluster. The Karmada scheduler can use this information to spread workloads across providers for higher availability.") flags.StringVar(&j.ClusterRegion, "cluster-region", "", "The region of the joining cluster. The Karmada scheduler can use this information to spread workloads across regions for higher availability.") flags.StringSliceVar(&j.ClusterZones, "cluster-zones", nil, "The zones of the joining cluster. The Karmada scheduler can use this information to spread workloads across zones for higher availability.") + flags.StringVar(&j.KarmadaAs, "karmada-as", "", + "Username to impersonate for the operation in karmada control plane. User could be a regular user or a service account in a namespace.") + flags.StringArrayVar(&j.KarmadaAsGroups, "karmada-as-group", []string{}, + "Group to impersonate for the operation in karmada control plane, this flag can be repeated to specify multiple groups.") + flags.StringVar(&j.KarmadaAsUID, "karmada-as-uid", "", + "UID to impersonate for the operation in karmada control plane.") flags.BoolVar(&j.DryRun, "dry-run", false, "Run the command in dry-run mode, without making any server requests.") } @@ -165,6 +180,11 @@ func (j *CommandJoinOption) Run(f cmdutil.Factory) error { *options.DefaultConfigFlags.Context, *options.DefaultConfigFlags.KubeConfig, err) } + // Configure impersonation + controlPlaneRestConfig.Impersonate.UserName = j.KarmadaAs + controlPlaneRestConfig.Impersonate.Groups = j.KarmadaAsGroups + controlPlaneRestConfig.Impersonate.UID = j.KarmadaAsUID + // Get cluster config clusterConfig, err := apiclient.RestConfig(j.ClusterContext, j.ClusterKubeConfig) if err != nil { diff --git a/pkg/registry/cluster/storage/aggregate.go b/pkg/registry/cluster/storage/aggregate.go index 7cec60f84346..c6c5136e3426 100644 --- a/pkg/registry/cluster/storage/aggregate.go +++ b/pkg/registry/cluster/storage/aggregate.go @@ -1,9 +1,12 @@ /* Copyright 2023 The Karmada Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/resourceinterpreter/customized/declarative/luavm/kube.go b/pkg/resourceinterpreter/customized/declarative/luavm/kube.go index 9af6173842cd..90e31454f23e 100644 --- a/pkg/resourceinterpreter/customized/declarative/luavm/kube.go +++ b/pkg/resourceinterpreter/customized/declarative/luavm/kube.go @@ -17,6 +17,8 @@ limitations under the License. package luavm import ( + "math" + lua "github.com/yuin/gopher-lua" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -57,6 +59,7 @@ var kubeFuncs = map[string]lua.LGFunction{ "resourceAdd": resourceAdd, "accuratePodRequirements": accuratePodRequirements, "getPodDependencies": getPodDependencies, + "getResourceQuantity": getResourceQuantity, } func resourceAdd(ls *lua.LState) int { @@ -127,6 +130,30 @@ func getPodDependencies(ls *lua.LState) int { return 1 } +func getResourceQuantity(ls *lua.LState) int { + n := ls.GetTop() + if n != 1 { + ls.RaiseError("getResourceQuantity only accepts one argument") + return 0 + } + + q := checkResourceQuantity(ls, n) + num := q.AsApproximateFloat64() + + if num < 0 { + ls.RaiseError("int approximation unexpectedly returned a negative value: %#v,", q) + return 0 + } + + if math.IsInf(num, 1) { + ls.RaiseError("int approximation unexpectedly returned an infinite value: %#v,", q) + return 0 + } + + ls.Push(lua.LNumber(num)) + return 1 +} + func checkResourceQuantity(ls *lua.LState, n int) resource.Quantity { v := ls.Get(n) switch typ := v.Type(); typ { diff --git a/pkg/resourceinterpreter/default/native/aggregatestatus.go b/pkg/resourceinterpreter/default/native/aggregatestatus.go index 561f75f021cc..3f7db8b5a883 100644 --- a/pkg/resourceinterpreter/default/native/aggregatestatus.go +++ b/pkg/resourceinterpreter/default/native/aggregatestatus.go @@ -65,27 +65,39 @@ func aggregateDeploymentStatus(object *unstructured.Unstructured, aggregatedStat oldStatus := &deploy.Status newStatus := &appsv1.DeploymentStatus{} + observedLatestResourceTemplateGenerationCount := 0 for _, item := range aggregatedStatusItems { if item.Status == nil { continue } - temp := &appsv1.DeploymentStatus{} - if err = json.Unmarshal(item.Status.Raw, temp); err != nil { + member := &WrappedDeploymentStatus{} + if err = json.Unmarshal(item.Status.Raw, member); err != nil { return nil, err } klog.V(3).Infof("Grab deployment(%s/%s) status from cluster(%s), replicas: %d, ready: %d, updated: %d, available: %d, unavailable: %d", - deploy.Namespace, deploy.Name, item.ClusterName, temp.Replicas, temp.ReadyReplicas, temp.UpdatedReplicas, temp.AvailableReplicas, temp.UnavailableReplicas) + deploy.Namespace, deploy.Name, item.ClusterName, member.Replicas, member.ReadyReplicas, member.UpdatedReplicas, member.AvailableReplicas, member.UnavailableReplicas) - // always set 'observedGeneration' with current generation(.metadata.generation) - // which is the generation Karmada 'observed'. - // The 'observedGeneration' is mainly used by GitOps tools(like 'Argo CD') to assess the health status. - // For more details, please refer to https://argo-cd.readthedocs.io/en/stable/operator-manual/health/. + // `memberStatus.ObservedGeneration >= memberStatus.Generation` means the member's status corresponds the latest spec revision of the member deployment. + // `memberStatus.ResourceTemplateGeneration >= deploy.Generation` means the member deployment has been aligned with the latest spec revision of federated deployment. + // If both conditions are met, we consider the member's status corresponds the latest spec revision of federated deployment. + if member.ObservedGeneration >= member.Generation && + member.ResourceTemplateGeneration >= deploy.Generation { + observedLatestResourceTemplateGenerationCount++ + } + + newStatus.Replicas += member.Replicas + newStatus.ReadyReplicas += member.ReadyReplicas + newStatus.UpdatedReplicas += member.UpdatedReplicas + newStatus.AvailableReplicas += member.AvailableReplicas + newStatus.UnavailableReplicas += member.UnavailableReplicas + } + + // The 'observedGeneration' is mainly used by GitOps tools(like 'Argo CD') to assess the health status. + // For more details, please refer to https://argo-cd.readthedocs.io/en/stable/operator-manual/health/. + if observedLatestResourceTemplateGenerationCount == len(aggregatedStatusItems) { newStatus.ObservedGeneration = deploy.Generation - newStatus.Replicas += temp.Replicas - newStatus.ReadyReplicas += temp.ReadyReplicas - newStatus.UpdatedReplicas += temp.UpdatedReplicas - newStatus.AvailableReplicas += temp.AvailableReplicas - newStatus.UnavailableReplicas += temp.UnavailableReplicas + } else { + newStatus.ObservedGeneration = oldStatus.ObservedGeneration } if oldStatus.ObservedGeneration == newStatus.ObservedGeneration && diff --git a/pkg/resourceinterpreter/default/native/reflectstatus.go b/pkg/resourceinterpreter/default/native/reflectstatus.go index d2da2214716d..408a8040e3d5 100644 --- a/pkg/resourceinterpreter/default/native/reflectstatus.go +++ b/pkg/resourceinterpreter/default/native/reflectstatus.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/klog/v2" + "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" "github.com/karmada-io/karmada/pkg/util" "github.com/karmada-io/karmada/pkg/util/helper" ) @@ -68,12 +69,25 @@ func reflectDeploymentStatus(object *unstructured.Unstructured) (*runtime.RawExt return nil, fmt.Errorf("failed to convert DeploymentStatus from map[string]interface{}: %v", err) } - grabStatus := appsv1.DeploymentStatus{ - Replicas: deploymentStatus.Replicas, - UpdatedReplicas: deploymentStatus.UpdatedReplicas, - ReadyReplicas: deploymentStatus.ReadyReplicas, - AvailableReplicas: deploymentStatus.AvailableReplicas, - UnavailableReplicas: deploymentStatus.UnavailableReplicas, + resourceTemplateGenerationInt := int64(0) + resourceTemplateGenerationStr := util.GetAnnotationValue(object.GetAnnotations(), v1alpha2.ResourceTemplateGenerationAnnotationKey) + err = runtime.Convert_string_To_int64(&resourceTemplateGenerationStr, &resourceTemplateGenerationInt, nil) + if err != nil { + klog.Errorf("Failed to parse Deployment(%s/%s) generation from annotation(%s:%s): %v", object.GetNamespace(), object.GetName(), v1alpha2.ResourceTemplateGenerationAnnotationKey, resourceTemplateGenerationStr, err) + return nil, err + } + + grabStatus := &WrappedDeploymentStatus{ + Generation: object.GetGeneration(), + ResourceTemplateGeneration: resourceTemplateGenerationInt, + DeploymentStatus: appsv1.DeploymentStatus{ + Replicas: deploymentStatus.Replicas, + UpdatedReplicas: deploymentStatus.UpdatedReplicas, + ReadyReplicas: deploymentStatus.ReadyReplicas, + AvailableReplicas: deploymentStatus.AvailableReplicas, + UnavailableReplicas: deploymentStatus.UnavailableReplicas, + ObservedGeneration: deploymentStatus.ObservedGeneration, + }, } grabStatusRaw, err := helper.BuildStatusRawExtension(grabStatus) diff --git a/pkg/resourceinterpreter/default/native/replica.go b/pkg/resourceinterpreter/default/native/replica.go index fbbc77bd3647..53b2a60038fc 100644 --- a/pkg/resourceinterpreter/default/native/replica.go +++ b/pkg/resourceinterpreter/default/native/replica.go @@ -23,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/klog/v2" + "k8s.io/utils/ptr" workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" "github.com/karmada-io/karmada/pkg/util" @@ -86,6 +87,13 @@ func jobReplica(object *unstructured.Unstructured) (int32, *workv1alpha2.Replica if job.Spec.Parallelism != nil { replica = *job.Spec.Parallelism } + // For fixed completion count Jobs, the actual number of pods running in parallel will not exceed the number of remaining completions. + // Higher values of .spec.parallelism are effectively ignored. + // More info: https://kubernetes.io/docs/concepts/workloads/controllers/job/ + completions := ptr.Deref[int32](job.Spec.Completions, replica) + if replica > completions { + replica = completions + } requirement := helper.GenerateReplicaRequirements(&job.Spec.Template) return replica, requirement, nil diff --git a/pkg/resourceinterpreter/default/native/status_type.go b/pkg/resourceinterpreter/default/native/status_type.go new file mode 100644 index 000000000000..9ba1b42a6032 --- /dev/null +++ b/pkg/resourceinterpreter/default/native/status_type.go @@ -0,0 +1,30 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package native + +import appsv1 "k8s.io/api/apps/v1" + +// WrappedDeploymentStatus is a wrapper for appsv1.DeploymentStatus. +type WrappedDeploymentStatus struct { + appsv1.DeploymentStatus `json:",inline"` + + // Generation holds the generation(.metadata.generation) of resource running on member cluster. + Generation int64 `json:"generation,omitempty"` + // ResourceTemplateGeneration holds the value of annotation resourcetemplate.karmada.io/generation grabbed + // from resource running on member cluster. + ResourceTemplateGeneration int64 `json:"resourceTemplateGeneration,omitempty"` +} diff --git a/pkg/resourceinterpreter/default/thirdparty/OWNERS b/pkg/resourceinterpreter/default/thirdparty/OWNERS index a90f231c69c5..64e11f751ce2 100644 --- a/pkg/resourceinterpreter/default/thirdparty/OWNERS +++ b/pkg/resourceinterpreter/default/thirdparty/OWNERS @@ -5,3 +5,4 @@ reviewers: approvers: - chaunceyjiang - Poor12 +- yike21 diff --git a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/apps.kruise.io/v1beta1/StatefulSet/customizations.yaml b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/apps.kruise.io/v1beta1/StatefulSet/customizations.yaml index 9b813a03646f..721a6fc55d6d 100644 --- a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/apps.kruise.io/v1beta1/StatefulSet/customizations.yaml +++ b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/apps.kruise.io/v1beta1/StatefulSet/customizations.yaml @@ -24,24 +24,46 @@ spec: statusAggregation: luaScript: > function AggregateStatus(desiredObj, statusItems) - if statusItems == nil then - return desiredObj - end if desiredObj.status == nil then desiredObj.status = {} end if desiredObj.metadata.generation == nil then desiredObj.metadata.generation = 0 end - generation = desiredObj.metadata.generation - replicas = 0 - readyReplicas = 0 - currentReplicas = 0 - updatedReplicas = 0 - availableReplicas = 0 - updatedReadyReplicas = 0 - updateRevision = '' - currentRevision = '' + if desiredObj.status.observedGeneration == nil then + desiredObj.status.observedGeneration = 0 + end + + -- Initialize status fields if status doest not exist + -- If the StatefulSet is not spread to any cluster, its status also should be aggregated + if statusItems == nil then + desiredObj.status.observedGeneration = desiredObj.metadata.generation + desiredObj.status.replicas = 0 + desiredObj.status.readyReplicas = 0 + desiredObj.status.currentReplicas = 0 + desiredObj.status.updatedReplicas = 0 + desiredObj.status.availableReplicas = 0 + desiredObj.status.updatedReadyReplicas = 0 + desiredObj.status.updateRevision = '' + desiredObj.status.currentRevision = '' + return desiredObj + end + + local generation = desiredObj.metadata.generation + local observedGeneration = desiredObj.status.observedGeneration + local replicas = 0 + local readyReplicas = 0 + local currentReplicas = 0 + local updatedReplicas = 0 + local availableReplicas = 0 + local updatedReadyReplicas = 0 + local updateRevision = '' + local currentRevision = '' + local labelSelector = '' + + -- Count all members that their status is updated to the latest generation + local observedResourceTemplateGenerationCount = 0 + for i = 1, #statusItems do if statusItems[i].status ~= nil and statusItems[i].status.replicas ~= nil then replicas = replicas + statusItems[i].status.replicas @@ -67,11 +89,32 @@ spec: if statusItems[i].status ~= nil and statusItems[i].status.currentRevision ~= nil and statusItems[i].status.currentRevision ~= '' then currentRevision = statusItems[i].status.currentRevision end - if statusItems[i].status ~= nil and statusItems[i].status.observedGeneration ~= nil and statusItems[i].status.observedGeneration ~= '' then - generation = statusItems[i].status.observedGeneration - end + + -- Check if the member's status is updated to the latest generation + local resourceTemplateGeneration = 0 + if statusItems[i].status ~= nil and statusItems[i].status.resourceTemplateGeneration ~= nil then + resourceTemplateGeneration = statusItems[i].status.resourceTemplateGeneration + end + local memberGeneration = 0 + if statusItems[i].status ~= nil and statusItems[i].status.generation ~= nil then + memberGeneration = statusItems[i].status.generation + end + local memberObservedGeneration = 0 + if statusItems[i].status ~= nil and statusItems[i].status.observedGeneration ~= nil then + memberObservedGeneration = statusItems[i].status.observedGeneration + end + if resourceTemplateGeneration == generation and memberGeneration == memberObservedGeneration then + observedResourceTemplateGenerationCount = observedResourceTemplateGenerationCount + 1 + end end - desiredObj.status.observedGeneration = generation + + -- Update the observed generation based on the observedResourceTemplateGenerationCount + if observedResourceTemplateGenerationCount == #statusItems then + desiredObj.status.observedGeneration = generation + else + desiredObj.status.observedGeneration = observedGeneration + end + desiredObj.status.replicas = replicas desiredObj.status.readyReplicas = readyReplicas desiredObj.status.currentReplicas = currentReplicas @@ -84,8 +127,8 @@ spec: end statusReflection: luaScript: > - function ReflectStatus (observedObj) - status = {} + function ReflectStatus(observedObj) + local status = {} if observedObj == nil or observedObj.status == nil then return status end @@ -98,6 +141,20 @@ spec: status.currentRevision = observedObj.status.currentRevision status.updatedReadyReplicas = observedObj.status.updatedReadyReplicas status.observedGeneration = observedObj.status.observedGeneration + + -- handle resource generation report + if observedObj.metadata == nil then + return status + end + status.generation = observedObj.metadata.generation + + if observedObj.metadata.annotations == nil then + return status + end + local resourceTemplateGeneration = tonumber(observedObj.metadata.annotations["resourcetemplate.karmada.io/generation"]) + if resourceTemplateGeneration ~= nil then + status.resourceTemplateGeneration = resourceTemplateGeneration + end return status end healthInterpretation: diff --git a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/apps.kruise.io/v1beta1/StatefulSet/testdata/desired-statefulset-nginx.yaml b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/apps.kruise.io/v1beta1/StatefulSet/testdata/desired-statefulset-nginx.yaml index b6478d42baff..47d506b45ac8 100644 --- a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/apps.kruise.io/v1beta1/StatefulSet/testdata/desired-statefulset-nginx.yaml +++ b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/apps.kruise.io/v1beta1/StatefulSet/testdata/desired-statefulset-nginx.yaml @@ -3,6 +3,7 @@ kind: StatefulSet metadata: name: sample namespace: test-statefulset + generation: 1 spec: replicas: 2 serviceName: sample-statefulset-headless-service diff --git a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/apps.kruise.io/v1beta1/StatefulSet/testdata/observed-statefulset-nginx.yaml b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/apps.kruise.io/v1beta1/StatefulSet/testdata/observed-statefulset-nginx.yaml index 2fa0aba9e44e..ccd464bf9ef1 100644 --- a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/apps.kruise.io/v1beta1/StatefulSet/testdata/observed-statefulset-nginx.yaml +++ b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/apps.kruise.io/v1beta1/StatefulSet/testdata/observed-statefulset-nginx.yaml @@ -1,6 +1,8 @@ apiVersion: apps.kruise.io/v1beta1 kind: StatefulSet metadata: + annotations: + resourcetemplate.karmada.io/generation: "1" name: sample namespace: test-statefulset generation: 1 diff --git a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/apps.kruise.io/v1beta1/StatefulSet/testdata/status-file.yaml b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/apps.kruise.io/v1beta1/StatefulSet/testdata/status-file.yaml index 1740b3ea3f92..76874392e502 100644 --- a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/apps.kruise.io/v1beta1/StatefulSet/testdata/status-file.yaml +++ b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/apps.kruise.io/v1beta1/StatefulSet/testdata/status-file.yaml @@ -5,6 +5,8 @@ status: availableReplicas: 2 currentReplicas: 2 currentRevision: sample-5675547df7 + generation: 1 + resourceTemplateGeneration: 1 observedGeneration: 1 readyReplicas: 2 replicas: 2 @@ -19,7 +21,9 @@ status: availableReplicas: 2 currentReplicas: 2 currentRevision: sample-5675547df7 + generation: 1 observedGeneration: 1 + resourceTemplateGeneration: 1 readyReplicas: 2 replicas: 2 updateRevision: sample-5675547df7 diff --git a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/flink.apache.org/v1beta1/FlinkDeployment/customizations.yaml b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/flink.apache.org/v1beta1/FlinkDeployment/customizations.yaml new file mode 100644 index 000000000000..9c326be56ea9 --- /dev/null +++ b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/flink.apache.org/v1beta1/FlinkDeployment/customizations.yaml @@ -0,0 +1,130 @@ +apiVersion: config.karmada.io/v1alpha1 +kind: ResourceInterpreterCustomization +metadata: + name: declarative-configuration-flinkdeployment +spec: + target: + apiVersion: flink.apache.org/v1beta1 + kind: FlinkDeployment + customizations: + healthInterpretation: + luaScript: > + function InterpretHealth(observedObj) + if observedObj.status ~= nil and observedObj.status.jobStatus ~= nil then + if observedObj.status.jobStatus.state ~= 'CREATED' and observedObj.status.jobStatus.state ~= 'RECONCILING' then + return true + else + return observedObj.status.jobManagerDeploymentStatus == 'ERROR' + end + end + return false + end + replicaResource: + luaScript: > + local kube = require("kube") + + local function isempty(s) + return s == nil or s == '' + end + + function GetReplicas(observedObj) + requires = { + resourceRequest = {}, + nodeClaim = {}, + } + + jm_replicas = observedObj.spec.jobManager.replicas + if isempty(jm_replicas) then + jm_replicas = 1 + end + + -- TaskManager replica setting takes precedence over parallelism setting + + tm_replicas = observedObj.spec.taskManager.replicas + if isempty(tm_replicas) then + parallelism = observedObj.spec.job.parallelism + task_slots = observedObj.spec.flinkConfiguration['taskmanager.numberOfTaskSlots'] + if isempty(parallelism) or isempty(task_slots) then + tm_replicas = 1 + else + tm_replicas = math.ceil(parallelism / observedObj.spec.flinkConfiguration['taskmanager.numberOfTaskSlots']) + end + end + + replica = jm_replicas + tm_replicas + + -- Until multiple podTemplates are supported in replicaRequirements, take max of cpu + memory values as requirement + + requires.resourceRequest.cpu = math.max(observedObj.spec.taskManager.resource.cpu, observedObj.spec.jobManager.resource.cpu) + jm_memory_value = kube.getResourceQuantity(observedObj.spec.jobManager.resource.memory) + tm_memory_value = kube.getResourceQuantity(observedObj.spec.taskManager.resource.memory) + if jm_memory_value > tm_memory_value then + requires.resourceRequest.memory = observedObj.spec.jobManager.resource.memory + else + requires.resourceRequest.memory = observedObj.spec.taskManager.resource.memory + end + + -- Until multiple podTemplates are supported, interpreter will only take affinity and toleration input to common podTemplate + + if observedObj.spec.podTemplate ~= nil and observedObj.spec.podTemplate.spec ~= nil then + requires.nodeClaim.nodeSelector = observedObj.spec.podTemplate.spec.nodeSelector + requires.nodeClaim.tolerations = observedObj.spec.podTemplate.spec.tolerations + end + + return replica, requires + end + statusAggregation: + luaScript: > + function AggregateStatus(desiredObj, statusItems) + if statusItems == nil then + return desiredObj + end + if desiredObj.status == nil then + desiredObj.status = {} + end + clusterInfo = {} + jobManagerDeploymentStatus = '' + jobStatus = {} + lifecycleState = '' + observedGeneration = 0 + reconciliationStatus = {} + taskManager = {} + + for i = 1, #statusItems do + currentStatus = statusItems[i].status + if currentStatus ~= nil then + clusterInfo = currentStatus.clusterInfo + jobManagerDeploymentStatus = currentStatus.jobManagerDeploymentStatus + jobStatus = currentStatus.jobStatus + observedGeneration = currentStatus.observedGeneration + lifecycleState = currentStatus.lifecycleState + reconciliationStatus = currentStatus.reconciliationStatus + taskManager = currentStatus.taskManager + end + end + + desiredObj.status.clusterInfo = clusterInfo + desiredObj.status.jobManagerDeploymentStatus = jobManagerDeploymentStatus + desiredObj.status.jobStatus = jobStatus + desiredObj.status.lifecycleState = lifecycleState + desiredObj.status.observedGeneration = observedGeneration + desiredObj.status.reconciliationStatus = reconciliationStatus + desiredObj.status.taskManager = taskManager + return desiredObj + end + statusReflection: + luaScript: > + function ReflectStatus(observedObj) + status = {} + if observedObj == nil or observedObj.status == nil then + return status + end + status.clusterInfo = observedObj.status.clusterInfo + status.jobManagerDeploymentStatus = observedObj.status.jobManagerDeploymentStatus + status.jobStatus = observedObj.status.jobStatus + status.observedGeneration = observedObj.status.observedGeneration + status.lifecycleState = observedObj.status.lifecycleState + status.reconciliationStatus = observedObj.status.reconciliationStatus + status.taskManager = observedObj.status.taskManager + return status + end diff --git a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/flink.apache.org/v1beta1/FlinkDeployment/customizations_tests.yaml b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/flink.apache.org/v1beta1/FlinkDeployment/customizations_tests.yaml new file mode 100644 index 000000000000..27c9f19e95cd --- /dev/null +++ b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/flink.apache.org/v1beta1/FlinkDeployment/customizations_tests.yaml @@ -0,0 +1,10 @@ +tests: + - desiredInputPath: testdata/desired-flinkdeployment.yaml + statusInputPath: testdata/status-file.yaml + operation: AggregateStatus + - observedInputPath: testdata/observed-flinkdeployment.yaml + operation: InterpretReplica + - observedInputPath: testdata/observed-flinkdeployment.yaml + operation: InterpretHealth + - observedInputPath: testdata/observed-flinkdeployment.yaml + operation: InterpretStatus diff --git a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/flink.apache.org/v1beta1/FlinkDeployment/testdata/desired-flinkdeployment.yaml b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/flink.apache.org/v1beta1/FlinkDeployment/testdata/desired-flinkdeployment.yaml new file mode 100644 index 000000000000..17b8dec57061 --- /dev/null +++ b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/flink.apache.org/v1beta1/FlinkDeployment/testdata/desired-flinkdeployment.yaml @@ -0,0 +1,25 @@ +apiVersion: flink.apache.org/v1beta1 +kind: FlinkDeployment +metadata: + name: basic-example + namespace: test-namespace +spec: + flinkConfiguration: + taskmanager.numberOfTaskSlots: "2" + flinkVersion: v1_17 + image: flink:1.17 + job: + jarURI: local:///opt/flink/examples/streaming/StateMachineExample.jar + parallelism: 2 + upgradeMode: stateless + jobManager: + replicas: 1 + resource: + cpu: 1 + memory: 2048m + mode: native + serviceAccount: flink + taskManager: + resource: + cpu: 1 + memory: 2048m diff --git a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/flink.apache.org/v1beta1/FlinkDeployment/testdata/observed-flinkdeployment.yaml b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/flink.apache.org/v1beta1/FlinkDeployment/testdata/observed-flinkdeployment.yaml new file mode 100644 index 000000000000..d31cf4eaa8f4 --- /dev/null +++ b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/flink.apache.org/v1beta1/FlinkDeployment/testdata/observed-flinkdeployment.yaml @@ -0,0 +1,60 @@ +apiVersion: flink.apache.org/v1beta1 +kind: FlinkDeployment +metadata: + creationTimestamp: "2024-06-05T14:52:28Z" + finalizers: + - flinkdeployments.flink.apache.org/finalizer + generation: 1 + name: basic-example + namespace: test-namespace + resourceVersion: "5053661" + uid: 87ef77ca-7bf0-4998-b275-06f459872e03 +spec: + flinkConfiguration: + taskmanager.numberOfTaskSlots: "2" + flinkVersion: v1_17 + image: flink:1.17 + job: + args: [] + jarURI: local:///opt/flink/examples/streaming/StateMachineExample.jar + parallelism: 2 + state: running + upgradeMode: stateless + jobManager: + replicas: 1 + resource: + cpu: 1 + memory: 2048m + serviceAccount: flink + taskManager: + resource: + cpu: 1 + memory: 2048m +status: + clusterInfo: + flink-revision: 2750d5c @ 2023-05-19T10:45:46+02:00 + flink-version: 1.17.1 + total-cpu: "2.0" + total-memory: "4294967296" + jobManagerDeploymentStatus: READY + jobStatus: + checkpointInfo: + lastPeriodicCheckpointTimestamp: 0 + jobId: 44cc5573945d1d4925732d915c70b9ac + jobName: Minimal Spec Example + savepointInfo: + lastPeriodicSavepointTimestamp: 0 + savepointHistory: [] + startTime: "1717599166365" + state: RUNNING + updateTime: "1717599182544" + lifecycleState: STABLE + observedGeneration: 1 + reconciliationStatus: + lastReconciledSpec: '{"spec":{"job":{"jarURI":"local:///opt/flink/examples/streaming/StateMachineExample.jar","parallelism":2,"entryClass":null,"args":[],"state":"running","savepointTriggerNonce":null,"initialSavepointPath":null,"checkpointTriggerNonce":null,"upgradeMode":"stateless","allowNonRestoredState":null,"savepointRedeployNonce":null},"restartNonce":null,"flinkConfiguration":{"taskmanager.numberOfTaskSlots":"2"},"image":"flink:1.17","imagePullPolicy":null,"serviceAccount":"flink","flinkVersion":"v1_17","ingress":null,"podTemplate":null,"jobManager":{"resource":{"cpu":1.0,"memory":"2048m","ephemeralStorage":null},"replicas":1,"podTemplate":null},"taskManager":{"resource":{"cpu":1.0,"memory":"2048m","ephemeralStorage":null},"replicas":null,"podTemplate":null},"logConfiguration":null,"mode":null},"resource_metadata":{"apiVersion":"flink.apache.org/v1beta1","metadata":{"generation":2},"firstDeployment":true}}' + lastStableSpec: '{"spec":{"job":{"jarURI":"local:///opt/flink/examples/streaming/StateMachineExample.jar","parallelism":2,"entryClass":null,"args":[],"state":"running","savepointTriggerNonce":null,"initialSavepointPath":null,"checkpointTriggerNonce":null,"upgradeMode":"stateless","allowNonRestoredState":null,"savepointRedeployNonce":null},"restartNonce":null,"flinkConfiguration":{"taskmanager.numberOfTaskSlots":"2"},"image":"flink:1.17","imagePullPolicy":null,"serviceAccount":"flink","flinkVersion":"v1_17","ingress":null,"podTemplate":null,"jobManager":{"resource":{"cpu":1.0,"memory":"2048m","ephemeralStorage":null},"replicas":1,"podTemplate":null},"taskManager":{"resource":{"cpu":1.0,"memory":"2048m","ephemeralStorage":null},"replicas":null,"podTemplate":null},"logConfiguration":null,"mode":null},"resource_metadata":{"apiVersion":"flink.apache.org/v1beta1","metadata":{"generation":2},"firstDeployment":true}}' + reconciliationTimestamp: 1717599148930 + state: DEPLOYED + taskManager: + labelSelector: component=taskmanager,app=basic-example + replicas: 1 diff --git a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/flink.apache.org/v1beta1/FlinkDeployment/testdata/status-file.yaml b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/flink.apache.org/v1beta1/FlinkDeployment/testdata/status-file.yaml new file mode 100644 index 000000000000..9a5c36808f5c --- /dev/null +++ b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/flink.apache.org/v1beta1/FlinkDeployment/testdata/status-file.yaml @@ -0,0 +1,31 @@ +applied: true +clusterName: member1 +health: Healthy +status: + clusterInfo: + flink-revision: 2750d5c @ 2023-05-19T10:45:46+02:00 + flink-version: 1.17.1 + total-cpu: "2.0" + total-memory: "4294967296" + jobManagerDeploymentStatus: READY + jobStatus: + checkpointInfo: + lastPeriodicCheckpointTimestamp: 0 + jobId: 44cc5573945d1d4925732d915c70b9ac + jobName: Minimal Spec Example + savepointInfo: + lastPeriodicSavepointTimestamp: 0 + savepointHistory: [] + startTime: "1717599166365" + state: RUNNING + updateTime: "1717599182544" + lifecycleState: STABLE + observedGeneration: 1 + reconciliationStatus: + lastReconciledSpec: '{"spec":{"job":{"jarURI":"local:///opt/flink/examples/streaming/StateMachineExample.jar","parallelism":2,"entryClass":null,"args":[],"state":"running","savepointTriggerNonce":null,"initialSavepointPath":null,"checkpointTriggerNonce":null,"upgradeMode":"stateless","allowNonRestoredState":null,"savepointRedeployNonce":null},"restartNonce":null,"flinkConfiguration":{"taskmanager.numberOfTaskSlots":"2"},"image":"flink:1.17","imagePullPolicy":null,"serviceAccount":"flink","flinkVersion":"v1_17","ingress":null,"podTemplate":null,"jobManager":{"resource":{"cpu":1.0,"memory":"2048m","ephemeralStorage":null},"replicas":1,"podTemplate":null},"taskManager":{"resource":{"cpu":1.0,"memory":"2048m","ephemeralStorage":null},"replicas":null,"podTemplate":null},"logConfiguration":null,"mode":null},"resource_metadata":{"apiVersion":"flink.apache.org/v1beta1","metadata":{"generation":2},"firstDeployment":true}}' + lastStableSpec: '{"spec":{"job":{"jarURI":"local:///opt/flink/examples/streaming/StateMachineExample.jar","parallelism":2,"entryClass":null,"args":[],"state":"running","savepointTriggerNonce":null,"initialSavepointPath":null,"checkpointTriggerNonce":null,"upgradeMode":"stateless","allowNonRestoredState":null,"savepointRedeployNonce":null},"restartNonce":null,"flinkConfiguration":{"taskmanager.numberOfTaskSlots":"2"},"image":"flink:1.17","imagePullPolicy":null,"serviceAccount":"flink","flinkVersion":"v1_17","ingress":null,"podTemplate":null,"jobManager":{"resource":{"cpu":1.0,"memory":"2048m","ephemeralStorage":null},"replicas":1,"podTemplate":null},"taskManager":{"resource":{"cpu":1.0,"memory":"2048m","ephemeralStorage":null},"replicas":null,"podTemplate":null},"logConfiguration":null,"mode":null},"resource_metadata":{"apiVersion":"flink.apache.org/v1beta1","metadata":{"generation":2},"firstDeployment":true}}' + reconciliationTimestamp: 1717599148930 + state: DEPLOYED + taskManager: + labelSelector: component=taskmanager,app=basic-example + replicas: 1 diff --git a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/kustomize.toolkit.fluxcd.io/v1/Kustomization/customizations.yaml b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/kustomize.toolkit.fluxcd.io/v1/Kustomization/customizations.yaml index edbb9fefe60f..4f658bbdb734 100644 --- a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/kustomize.toolkit.fluxcd.io/v1/Kustomization/customizations.yaml +++ b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/kustomize.toolkit.fluxcd.io/v1/Kustomization/customizations.yaml @@ -22,22 +22,42 @@ spec: statusAggregation: luaScript: > function AggregateStatus(desiredObj, statusItems) + if desiredObj.status == nil then + desiredObj.status = {} + end + if desiredObj.metadata.generation == nil then + desiredObj.metadata.generation = 0 + end + if desiredObj.status.observedGeneration == nil then + desiredObj.status.observedGeneration = 0 + end + + -- Initialize status fields if status doest not exist + -- If the Kustomization is not spread to any cluster, its status also should be aggregated if statusItems == nil then + desiredObj.status.observedGeneration = desiredObj.metadata.generation + desiredObj.status.lastAttemptedRevision = '' + desiredObj.status.lastAppliedRevision = '' + desiredObj.status.conditions = {} return desiredObj end - desiredObj.status = {} - desiredObj.status.conditions = {} - conditions = {} + + local conditions = {} + local generation = desiredObj.metadata.generation + local lastAppliedRevision = desiredObj.status.lastAppliedRevision + local lastAttemptedRevision = desiredObj.status.lastAttemptedRevision + local observedGeneration = desiredObj.status.observedGeneration + + -- Count all members that their status is updated to the latest generation + local observedResourceTemplateGenerationCount = 0 + local conditionsIndex = 1 for i = 1, #statusItems do if statusItems[i].status ~= nil and statusItems[i].status.lastAttemptedRevision ~= nil and statusItems[i].status.lastAttemptedRevision ~= '' then - desiredObj.status.lastAttemptedRevision = statusItems[i].status.lastAttemptedRevision + lastAttemptedRevision = statusItems[i].status.lastAttemptedRevision end if statusItems[i].status ~= nil and statusItems[i].status.lastAppliedRevision ~= nil and statusItems[i].status.lastAppliedRevision ~= '' then - desiredObj.status.lastAppliedRevision = statusItems[i].status.lastAppliedRevision - end - if statusItems[i].status ~= nil and statusItems[i].status.lastHandledReconcileAt ~= nil and statusItems[i].status.lastHandledReconcileAt ~= '' then - desiredObj.status.lastHandledReconcileAt = statusItems[i].status.lastHandledReconcileAt + lastAppliedRevision = statusItems[i].status.lastAppliedRevision end if statusItems[i].status ~= nil and statusItems[i].status.conditions ~= nil then for conditionIndex = 1, #statusItems[i].status.conditions do @@ -56,9 +76,35 @@ spec: end end end + + -- Check if the member's status is updated to the latest generation + local resourceTemplateGeneration = 0 + if statusItems[i].status ~= nil and statusItems[i].status.resourceTemplateGeneration ~= nil then + resourceTemplateGeneration = statusItems[i].status.resourceTemplateGeneration + end + local memberGeneration = 0 + if statusItems[i].status ~= nil and statusItems[i].status.generation ~= nil then + memberGeneration = statusItems[i].status.generation + end + local memberObservedGeneration = 0 + if statusItems[i].status ~= nil and statusItems[i].status.observedGeneration ~= nil then + memberObservedGeneration = statusItems[i].status.observedGeneration + end + if resourceTemplateGeneration == generation and memberGeneration == memberObservedGeneration then + observedResourceTemplateGenerationCount = observedResourceTemplateGenerationCount + 1 + end end - desiredObj.status.observedGeneration = desiredObj.metadata.generation + + -- Update the observed generation based on the observedResourceTemplateGenerationCount + if observedResourceTemplateGenerationCount == #statusItems then + desiredObj.status.observedGeneration = generation + else + desiredObj.status.observedGeneration = observedGeneration + end + desiredObj.status.conditions = conditions + desiredObj.status.lastAppliedRevision = lastAppliedRevision + desiredObj.status.lastAttemptedRevision = lastAttemptedRevision return desiredObj end retention: @@ -72,14 +118,29 @@ spec: statusReflection: luaScript: > function ReflectStatus (observedObj) - status = {} - if observedObj == nil or observedObj.status == nil then + local status = {} + if observedObj == nil or observedObj.status == nil then return status end status.conditions = observedObj.status.conditions status.lastAppliedRevision = observedObj.status.lastAppliedRevision status.lastAttemptedRevision = observedObj.status.lastAttemptedRevision - status.lastHandledReconcileAt = observedObj.status.lastHandledReconcileAt + status.observedGeneration = observedObj.status.observedGeneration + + -- handle resource generation report + if observedObj.metadata == nil then + return status + end + status.generation = observedObj.metadata.generation + + if observedObj.metadata.annotations == nil then + return status + end + local resourceTemplateGeneration = tonumber(observedObj.metadata.annotations["resourcetemplate.karmada.io/generation"]) + if resourceTemplateGeneration ~= nil then + status.resourceTemplateGeneration = resourceTemplateGeneration + end + return status end dependencyInterpretation: diff --git a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/kustomize.toolkit.fluxcd.io/v1/Kustomization/testdata/desired-kustomization.yaml b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/kustomize.toolkit.fluxcd.io/v1/Kustomization/testdata/desired-kustomization.yaml index 27531a4b3909..4c2b6a47d0d0 100644 --- a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/kustomize.toolkit.fluxcd.io/v1/Kustomization/testdata/desired-kustomization.yaml +++ b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/kustomize.toolkit.fluxcd.io/v1/Kustomization/testdata/desired-kustomization.yaml @@ -3,6 +3,7 @@ kind: Kustomization metadata: name: sample namespace: test-kustomization + generation: 1 spec: interval: 10m targetNamespace: test-kustomization diff --git a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/kustomize.toolkit.fluxcd.io/v1/Kustomization/testdata/observed-kustomization.yaml b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/kustomize.toolkit.fluxcd.io/v1/Kustomization/testdata/observed-kustomization.yaml index d0f960912fd1..c89012914ebd 100644 --- a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/kustomize.toolkit.fluxcd.io/v1/Kustomization/testdata/observed-kustomization.yaml +++ b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/kustomize.toolkit.fluxcd.io/v1/Kustomization/testdata/observed-kustomization.yaml @@ -1,8 +1,11 @@ apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: + annotations: + resourcetemplate.karmada.io/generation: "1" name: sample namespace: test-kustomization + generation: 1 spec: interval: 10m targetNamespace: test-kustomization diff --git a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/kustomize.toolkit.fluxcd.io/v1/Kustomization/testdata/status-file.yaml b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/kustomize.toolkit.fluxcd.io/v1/Kustomization/testdata/status-file.yaml index f0cfa3ac4dd0..608faa3e85d3 100644 --- a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/kustomize.toolkit.fluxcd.io/v1/Kustomization/testdata/status-file.yaml +++ b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/kustomize.toolkit.fluxcd.io/v1/Kustomization/testdata/status-file.yaml @@ -9,8 +9,11 @@ status: reason: ReconciliationSucceeded status: "True" type: Ready + generation: 1 lastAppliedRevision: master@sha1:0647aea75b85755411b007a290b9321668370be5 lastAttemptedRevision: master@sha1:0647aea75b85755411b007a290b9321668370be5 + observedGeneration: 1 + resourceTemplateGeneration: 1 --- applied: true clusterName: member3 @@ -23,5 +26,8 @@ status: reason: ReconciliationSucceeded status: "True" type: Ready + generation: 1 lastAppliedRevision: master@sha1:0647aea75b85755411b007a290b9321668370be5 lastAttemptedRevision: master@sha1:0647aea75b85755411b007a290b9321668370be5 + observedGeneration: 1 + resourceTemplateGeneration: 1 diff --git a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/source.toolkit.fluxcd.io/v1/GitRepository/customizations.yaml b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/source.toolkit.fluxcd.io/v1/GitRepository/customizations.yaml index ace6ff3f71f2..14c99d73b02a 100644 --- a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/source.toolkit.fluxcd.io/v1/GitRepository/customizations.yaml +++ b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/source.toolkit.fluxcd.io/v1/GitRepository/customizations.yaml @@ -22,19 +22,37 @@ spec: statusAggregation: luaScript: > function AggregateStatus(desiredObj, statusItems) + if desiredObj.status == nil then + desiredObj.status = {} + end + if desiredObj.metadata.generation == nil then + desiredObj.metadata.generation = 0 + end + if desiredObj.status.observedGeneration == nil then + desiredObj.status.observedGeneration = 0 + end + + -- Initialize status fields if status doest not exist + -- If the GitRepository is not spread to any cluster, its status also should be aggregated if statusItems == nil then + desiredObj.status.artifact = {} + desiredObj.status.conditions = {} + desiredObj.status.observedGeneration = desiredObj.metadata.generation return desiredObj end - desiredObj.status = {} - desiredObj.status.conditions = {} - conditions = {} + + local artifact = {} + local conditions = {} local conditionsIndex = 1 + local generation = desiredObj.metadata.generation + local observedGeneration = desiredObj.status.observedGeneration + + -- Count all members that their status is updated to the latest generation + local observedResourceTemplateGenerationCount = 0 + for i = 1, #statusItems do if statusItems[i].status ~= nil and statusItems[i].status.artifact ~= nil then - desiredObj.status.artifact = statusItems[i].status.artifact - end - if statusItems[i].status ~= nil and statusItems[i].status.lastHandledReconcileAt ~= nil and statusItems[i].status.lastHandledReconcileAt ~='' then - desiredObj.status.lastHandledReconcileAt = statusItems[i].status.lastHandledReconcileAt + artifact = statusItems[i].status.artifact end if statusItems[i].status ~= nil and statusItems[i].status.conditions ~= nil then for conditionIndex = 1, #statusItems[i].status.conditions do @@ -53,8 +71,33 @@ spec: end end end + + -- Check if the member's status is updated to the latest generation + local resourceTemplateGeneration = 0 + if statusItems[i].status ~= nil and statusItems[i].status.resourceTemplateGeneration ~= nil then + resourceTemplateGeneration = statusItems[i].status.resourceTemplateGeneration + end + local memberGeneration = 0 + if statusItems[i].status ~= nil and statusItems[i].status.generation ~= nil then + memberGeneration = statusItems[i].status.generation + end + local memberObservedGeneration = 0 + if statusItems[i].status ~= nil and statusItems[i].status.observedGeneration ~= nil then + memberObservedGeneration = statusItems[i].status.observedGeneration + end + if resourceTemplateGeneration == generation and memberGeneration == memberObservedGeneration then + observedResourceTemplateGenerationCount = observedResourceTemplateGenerationCount + 1 + end + end + + -- Update the observed generation based on the observedResourceTemplateGenerationCount + if observedResourceTemplateGenerationCount == #statusItems then + desiredObj.status.observedGeneration = generation + else + desiredObj.status.observedGeneration = observedGeneration end - desiredObj.status.observedGeneration = desiredObj.metadata.generation + + desiredObj.status.artifact = artifact desiredObj.status.conditions = conditions return desiredObj end @@ -69,15 +112,29 @@ spec: statusReflection: luaScript: > function ReflectStatus (observedObj) - status = {} - if observedObj == nil or observedObj.status == nil then + local status = {} + if observedObj == nil or observedObj.status == nil then return status end status.conditions = observedObj.status.conditions status.artifact = observedObj.status.artifact + status.observedGeneration = observedObj.status.observedGeneration status.observedIgnore = observedObj.status.observedIgnore status.observedRecurseSubmodules = observedObj.status.observedRecurseSubmodules - status.lastHandledReconcileAt = observedObj.status.lastHandledReconcileAt + + -- handle resource generation report + if observedObj.metadata == nil then + return status + end + status.generation = observedObj.metadata.generation + if observedObj.metadata.annotations == nil then + return status + end + local resourceTemplateGeneration = tonumber(observedObj.metadata.annotations["resourcetemplate.karmada.io/generation"]) + if resourceTemplateGeneration ~= nil then + status.resourceTemplateGeneration = resourceTemplateGeneration + end + return status end dependencyInterpretation: diff --git a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/source.toolkit.fluxcd.io/v1/GitRepository/testdata/desired-gitrepository.yaml b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/source.toolkit.fluxcd.io/v1/GitRepository/testdata/desired-gitrepository.yaml index cb16af999514..f0c3e95e7ef6 100644 --- a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/source.toolkit.fluxcd.io/v1/GitRepository/testdata/desired-gitrepository.yaml +++ b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/source.toolkit.fluxcd.io/v1/GitRepository/testdata/desired-gitrepository.yaml @@ -1,6 +1,7 @@ apiVersion: source.toolkit.fluxcd.io/v1 kind: GitRepository metadata: + generation: 1 name: sample namespace: test-gitrepository spec: diff --git a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/source.toolkit.fluxcd.io/v1/GitRepository/testdata/observed-gitrepository.yaml b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/source.toolkit.fluxcd.io/v1/GitRepository/testdata/observed-gitrepository.yaml index 0467da533074..1a64811afe14 100644 --- a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/source.toolkit.fluxcd.io/v1/GitRepository/testdata/observed-gitrepository.yaml +++ b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/source.toolkit.fluxcd.io/v1/GitRepository/testdata/observed-gitrepository.yaml @@ -1,6 +1,9 @@ apiVersion: source.toolkit.fluxcd.io/v1 kind: GitRepository metadata: + annotations: + resourcetemplate.karmada.io/generation: "1" + generation: 1 name: sample namespace: test-gitrepository spec: diff --git a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/source.toolkit.fluxcd.io/v1/GitRepository/testdata/status-file.yaml b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/source.toolkit.fluxcd.io/v1/GitRepository/testdata/status-file.yaml index 87a7b8eeba08..6878cd4bfbb2 100644 --- a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/source.toolkit.fluxcd.io/v1/GitRepository/testdata/status-file.yaml +++ b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/source.toolkit.fluxcd.io/v1/GitRepository/testdata/status-file.yaml @@ -22,6 +22,9 @@ status: reason: Succeeded status: "True" type: ArtifactInStorage + generation: 1 + observedGeneration: 1 + resourceTemplateGeneration: 1 --- applied: true clusterName: member3 @@ -47,3 +50,6 @@ status: reason: Succeeded status: "True" type: ArtifactInStorage + generation: 1 + observedGeneration: 1 + resourceTemplateGeneration: 1 diff --git a/pkg/scheduler/OWNERS b/pkg/scheduler/OWNERS index c84fa0b67361..557df168f6a1 100644 --- a/pkg/scheduler/OWNERS +++ b/pkg/scheduler/OWNERS @@ -9,4 +9,5 @@ reviewers: - XiShanYongYe-Chang approvers: - Garrybest +- whitewindmills - XiShanYongYe-Chang diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 2f4e7737d895..0e9f0a3dc78b 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -56,6 +56,7 @@ import ( "github.com/karmada-io/karmada/pkg/scheduler/metrics" "github.com/karmada-io/karmada/pkg/sharedcli/ratelimiterflag" "github.com/karmada-io/karmada/pkg/util" + "github.com/karmada-io/karmada/pkg/util/grpcconnection" "github.com/karmada-io/karmada/pkg/util/helper" utilmetrics "github.com/karmada-io/karmada/pkg/util/metrics" ) @@ -106,8 +107,8 @@ type Scheduler struct { disableSchedulerEstimatorInPullMode bool schedulerEstimatorCache *estimatorclient.SchedulerEstimatorCache schedulerEstimatorServicePrefix string - schedulerEstimatorPort int schedulerEstimatorWorker util.AsyncWorker + schedulerEstimatorClientConfig *grpcconnection.ClientConfig schedulerName string enableEmptyWorkloadPropagation bool @@ -122,8 +123,6 @@ type schedulerOptions struct { schedulerEstimatorTimeout metav1.Duration // SchedulerEstimatorServicePrefix presents the prefix of the accurate scheduler estimator service name. schedulerEstimatorServicePrefix string - // schedulerEstimatorPort is the port that the accurate scheduler estimator server serves at. - schedulerEstimatorPort int // schedulerName is the name of the scheduler. Default is "default-scheduler". schedulerName string //enableEmptyWorkloadPropagation represents whether allow workload with replicas 0 propagated to member clusters should be enabled @@ -134,6 +133,8 @@ type schedulerOptions struct { plugins []string // contains the options for rate limiter. RateLimiterOptions ratelimiterflag.Options + // schedulerEstimatorClientConfig contains the configuration of GRPC. + schedulerEstimatorClientConfig *grpcconnection.ClientConfig } // Option configures a Scheduler @@ -146,6 +147,19 @@ func WithEnableSchedulerEstimator(enableSchedulerEstimator bool) Option { } } +// WithSchedulerEstimatorConnection sets the grpc config for scheduler +func WithSchedulerEstimatorConnection(port int, certFile, keyFile, trustedCAFile string, insecureSkipVerify bool) Option { + return func(o *schedulerOptions) { + o.schedulerEstimatorClientConfig = &grpcconnection.ClientConfig{ + CertFile: certFile, + KeyFile: keyFile, + ServerAuthCAFile: trustedCAFile, + InsecureSkipServerVerify: insecureSkipVerify, + TargetPort: port, + } + } +} + // WithDisableSchedulerEstimatorInPullMode sets the disableSchedulerEstimatorInPullMode for scheduler func WithDisableSchedulerEstimatorInPullMode(disableSchedulerEstimatorInPullMode bool) Option { return func(o *schedulerOptions) { @@ -167,13 +181,6 @@ func WithSchedulerEstimatorServicePrefix(schedulerEstimatorServicePrefix string) } } -// WithSchedulerEstimatorPort sets the schedulerEstimatorPort for scheduler -func WithSchedulerEstimatorPort(schedulerEstimatorPort int) Option { - return func(o *schedulerOptions) { - o.schedulerEstimatorPort = schedulerEstimatorPort - } -} - // WithSchedulerName sets the schedulerName for scheduler func WithSchedulerName(schedulerName string) Option { return func(o *schedulerOptions) { @@ -255,7 +262,7 @@ func NewScheduler(dynamicClient dynamic.Interface, karmadaClient karmadaclientse sched.enableSchedulerEstimator = options.enableSchedulerEstimator sched.disableSchedulerEstimatorInPullMode = options.disableSchedulerEstimatorInPullMode sched.schedulerEstimatorServicePrefix = options.schedulerEstimatorServicePrefix - sched.schedulerEstimatorPort = options.schedulerEstimatorPort + sched.schedulerEstimatorClientConfig = options.schedulerEstimatorClientConfig sched.schedulerEstimatorCache = estimatorclient.NewSchedulerEstimatorCache() schedulerEstimatorWorkerOptions := util.Options{ Name: "scheduler-estimator", @@ -769,7 +776,7 @@ func (s *Scheduler) reconcileEstimatorConnection(key util.QueueKey) error { return nil } - return estimatorclient.EstablishConnection(s.KubeClient, name, s.schedulerEstimatorCache, s.schedulerEstimatorServicePrefix, s.schedulerEstimatorPort) + return estimatorclient.EstablishConnection(s.KubeClient, name, s.schedulerEstimatorCache, s.schedulerEstimatorServicePrefix, s.schedulerEstimatorClientConfig) } func (s *Scheduler) establishEstimatorConnections() { @@ -782,7 +789,7 @@ func (s *Scheduler) establishEstimatorConnections() { if clusterList.Items[i].Spec.SyncMode == clusterv1alpha1.Pull && s.disableSchedulerEstimatorInPullMode { continue } - if err = estimatorclient.EstablishConnection(s.KubeClient, clusterList.Items[i].Name, s.schedulerEstimatorCache, s.schedulerEstimatorServicePrefix, s.schedulerEstimatorPort); err != nil { + if err = estimatorclient.EstablishConnection(s.KubeClient, clusterList.Items[i].Name, s.schedulerEstimatorCache, s.schedulerEstimatorServicePrefix, s.schedulerEstimatorClientConfig); err != nil { klog.Error(err) } } diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 584eb226e224..da83e28fec2b 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -53,16 +53,16 @@ func TestCreateScheduler(t *testing.T) { name: "scheduler with enableSchedulerEstimator enabled", opts: []Option{ WithEnableSchedulerEstimator(true), - WithSchedulerEstimatorPort(port), + WithSchedulerEstimatorConnection(port, "", "", "", false), }, enableSchedulerEstimator: true, schedulerEstimatorPort: port, }, { - name: "scheduler with enableSchedulerEstimator disabled, WithSchedulerEstimatorPort enabled", + name: "scheduler with enableSchedulerEstimator disabled, WithSchedulerEstimatorConnection enabled", opts: []Option{ WithEnableSchedulerEstimator(false), - WithSchedulerEstimatorPort(port), + WithSchedulerEstimatorConnection(port, "", "", "", false), }, enableSchedulerEstimator: false, }, @@ -79,8 +79,8 @@ func TestCreateScheduler(t *testing.T) { t.Errorf("unexpected enableSchedulerEstimator want %v, got %v", tc.enableSchedulerEstimator, sche.enableSchedulerEstimator) } - if tc.schedulerEstimatorPort != sche.schedulerEstimatorPort { - t.Errorf("unexpected schedulerEstimatorPort want %v, got %v", tc.schedulerEstimatorPort, sche.schedulerEstimatorPort) + if tc.enableSchedulerEstimator && tc.schedulerEstimatorPort != sche.schedulerEstimatorClientConfig.TargetPort { + t.Errorf("unexpected schedulerEstimatorPort want %v, got %v", tc.schedulerEstimatorPort, sche.schedulerEstimatorClientConfig.TargetPort) } }) } diff --git a/pkg/search/proxy/framework/plugins/cluster/cluster.go b/pkg/search/proxy/framework/plugins/cluster/cluster.go index 5128020ac731..9434fb7e3330 100644 --- a/pkg/search/proxy/framework/plugins/cluster/cluster.go +++ b/pkg/search/proxy/framework/plugins/cluster/cluster.go @@ -150,8 +150,7 @@ func modifyRequest(req *http.Request, cluster string) error { return nil } - changed := false - changed = store.RemoveCacheSourceAnnotation(obj) || changed + changed := store.RemoveCacheSourceAnnotation(obj) changed = store.RecoverClusterResourceVersion(obj, cluster) || changed if changed { diff --git a/pkg/util/dial.go b/pkg/util/dial.go deleted file mode 100644 index ab12bcda5ee1..000000000000 --- a/pkg/util/dial.go +++ /dev/null @@ -1,43 +0,0 @@ -/* -Copyright 2021 The Karmada Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "context" - "fmt" - "time" - - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" -) - -// Dial establishes the gRPC communication. -func Dial(path string, timeout time.Duration) (*grpc.ClientConn, error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - opts := []grpc.DialOption{ - grpc.WithBlock(), - grpc.WithTransportCredentials(insecure.NewCredentials()), - } - cc, err := grpc.DialContext(ctx, path, opts...) - if err != nil { - return nil, fmt.Errorf("dial %s error: %v", path, err) - } - - return cc, nil -} diff --git a/pkg/util/grpcconnection/config.go b/pkg/util/grpcconnection/config.go new file mode 100644 index 000000000000..149d0aeefa4e --- /dev/null +++ b/pkg/util/grpcconnection/config.go @@ -0,0 +1,147 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package grpcconnection + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "os" + "time" + + "google.golang.org/grpc" + grpccredentials "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" +) + +// ServerConfig the config of GRPC server side. +type ServerConfig struct { + // ServerPort The secure port on which to serve gRPC. + ServerPort int + // InsecureSkipClientVerify Controls whether verifies the client's certificate chain and host name. + // When this is set to false, server will check all incoming HTTPS requests for a client certificate signed by the trusted CA, + // requests that don’t supply a valid client certificate will fail. If authentication is enabled, + // the certificate provides credentials for the user name given by the Common Name field. + InsecureSkipClientVerify bool + // ClientAuthCAFile SSL Certificate Authority file used to verify grpc client certificates on incoming requests. + ClientAuthCAFile string + // CertFile SSL certification file used for grpc SSL/TLS connections. + CertFile string + // KeyFile SSL key file used for grpc SSL/TLS connections. + KeyFile string +} + +// ClientConfig the config of GRPC client side. +type ClientConfig struct { + // TargetPort the target port on which to establish a gRPC connection. + TargetPort int + // InsecureSkipServerVerify controls whether a client verifies the server's + // certificate chain and host name. If InsecureSkipServerVerify is true, crypto/tls + // accepts any certificate presented by the server and any host name in that + // certificate. In this mode, TLS is susceptible to machine-in-the-middle + // attacks unless custom verification is used. This should be used only for + // testing or in combination with VerifyConnection or VerifyPeerCertificate. + InsecureSkipServerVerify bool + // ServerAuthCAFile SSL Certificate Authority file used to verify grpc server certificates. + ServerAuthCAFile string + // SSL certification file used for grpc SSL/TLS connections. + CertFile string + // SSL key file used for grpc SSL/TLS connections. + KeyFile string +} + +// NewServer creates a gRPC server which has no service registered and has not +// started to accept requests yet. +func (s *ServerConfig) NewServer() (*grpc.Server, error) { + if s.CertFile == "" || s.KeyFile == "" { + return grpc.NewServer(), nil + } + + cert, err := tls.LoadX509KeyPair(s.CertFile, s.KeyFile) + if err != nil { + return nil, err + } + config := &tls.Config{ + Certificates: []tls.Certificate{cert}, + MinVersion: tls.VersionTLS13, + } + + if s.ClientAuthCAFile != "" { + certPool := x509.NewCertPool() + ca, err := os.ReadFile(s.ClientAuthCAFile) + if err != nil { + return nil, err + } + if ok := certPool.AppendCertsFromPEM(ca); !ok { + return nil, fmt.Errorf("failed to append ca into certPool") + } + config.ClientCAs = certPool + if !s.InsecureSkipClientVerify { + config.ClientAuth = tls.RequireAndVerifyClientCert + } + } + + return grpc.NewServer(grpc.Creds(grpccredentials.NewTLS(config))), nil +} + +// DialWithTimeOut creates a client connection to the given target. +func (c *ClientConfig) DialWithTimeOut(path string, timeout time.Duration) (*grpc.ClientConn, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + opts := []grpc.DialOption{ + grpc.WithBlock(), + } + + var cred grpccredentials.TransportCredentials + if c.ServerAuthCAFile == "" && !c.InsecureSkipServerVerify { + // insecure connection + cred = insecure.NewCredentials() + } else { + // server-side TLS + config := &tls.Config{InsecureSkipVerify: c.InsecureSkipServerVerify} // nolint:gosec // G402: TLS InsecureSkipEstimatorVerify may be true. + if c.ServerAuthCAFile != "" { + certPool := x509.NewCertPool() + ca, err := os.ReadFile(c.ServerAuthCAFile) + if err != nil { + return nil, err + } + if ok := certPool.AppendCertsFromPEM(ca); !ok { + return nil, fmt.Errorf("failed to append ca certs") + } + config.RootCAs = certPool + } + if c.CertFile != "" && c.KeyFile != "" { + // mutual TLS + certificate, err := tls.LoadX509KeyPair(c.CertFile, c.KeyFile) + if err != nil { + return nil, err + } + config.Certificates = []tls.Certificate{certificate} + } + cred = grpccredentials.NewTLS(config) + } + + opts = append(opts, grpc.WithTransportCredentials(cred)) + cc, err := grpc.DialContext(ctx, path, opts...) + if err != nil { + return nil, fmt.Errorf("dial %s error: %v", path, err) + } + + return cc, nil +} diff --git a/pkg/util/hash/hash.go b/pkg/util/hash/hash.go index 31facf6c2f00..dd21a922015a 100644 --- a/pkg/util/hash/hash.go +++ b/pkg/util/hash/hash.go @@ -27,5 +27,5 @@ import ( // ensuring the hash does not change when a pointer changes. func DeepHashObject(hasher hash.Hash, objectToWrite interface{}) { hasher.Reset() - pretty.Fprintf(hasher, "%# v", objectToWrite) + pretty.Fprintf(hasher, "%v", objectToWrite) } diff --git a/pkg/util/helper/cronfederatedhpa.go b/pkg/util/helper/cronfederatedhpa.go index d3659d4b320a..639d5d2aba22 100644 --- a/pkg/util/helper/cronfederatedhpa.go +++ b/pkg/util/helper/cronfederatedhpa.go @@ -1,5 +1,6 @@ /* Copyright 2023 The Karmada Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at diff --git a/pkg/util/helper/status.go b/pkg/util/helper/status.go new file mode 100755 index 000000000000..2a41df6f81af --- /dev/null +++ b/pkg/util/helper/status.go @@ -0,0 +1,69 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helper + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/api/equality" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// UpdateStatus updates the given object's status in the Kubernetes +// cluster. The object's desired state must be reconciled with the existing +// state inside the passed in callback MutateFn. +// +// The MutateFn is called when updating an object's status. +// +// It returns the executed operation and an error. +// +// Note: changes to any sub-resource other than status will be ignored. +// Changes to the status sub-resource will only be applied if the object +// already exist. +func UpdateStatus(ctx context.Context, c client.Client, obj client.Object, f controllerutil.MutateFn) (controllerutil.OperationResult, error) { + key := client.ObjectKeyFromObject(obj) + if err := c.Get(ctx, key, obj); err != nil { + return controllerutil.OperationResultNone, err + } + + existing := obj.DeepCopyObject() + if err := mutate(f, key, obj); err != nil { + return controllerutil.OperationResultNone, err + } + + if equality.Semantic.DeepEqual(existing, obj) { + return controllerutil.OperationResultNone, nil + } + + if err := c.Status().Update(ctx, obj); err != nil { + return controllerutil.OperationResultNone, err + } + return controllerutil.OperationResultUpdatedStatusOnly, nil +} + +// mutate wraps a MutateFn and applies validation to its result. +func mutate(f controllerutil.MutateFn, key client.ObjectKey, obj client.Object) error { + if err := f(); err != nil { + return err + } + if newKey := client.ObjectKeyFromObject(obj); key != newKey { + return fmt.Errorf("MutateFn cannot mutate object name and/or object namespace") + } + return nil +} diff --git a/pkg/util/helper/workstatus.go b/pkg/util/helper/workstatus.go index ca40aef2cdd9..8f3a275f5bf8 100644 --- a/pkg/util/helper/workstatus.go +++ b/pkg/util/helper/workstatus.go @@ -20,7 +20,6 @@ import ( "context" "encoding/json" "fmt" - "reflect" "sort" corev1 "k8s.io/api/core/v1" @@ -34,6 +33,7 @@ import ( "k8s.io/client-go/util/retry" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" workv1alpha1 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha1" workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" @@ -73,41 +73,28 @@ func AggregateResourceBindingWorkStatus( fullyAppliedCondition := generateFullyAppliedCondition(binding.Spec, aggregatedStatuses) - err = retry.RetryOnConflict(retry.DefaultRetry, func() (err error) { - currentBindingStatus := binding.Status.DeepCopy() - - binding.Status.AggregatedStatus = aggregatedStatuses - // set binding status with the newest condition - meta.SetStatusCondition(&binding.Status.Conditions, fullyAppliedCondition) - if reflect.DeepEqual(binding.Status, *currentBindingStatus) { - klog.V(4).Infof("New aggregatedStatuses are equal with old resourceBinding(%s/%s) AggregatedStatus, no update required.", - binding.Namespace, binding.Name) - return nil - } - - updateErr := c.Status().Update(context.TODO(), binding) - if updateErr == nil { + var operationResult controllerutil.OperationResult + if err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + operationResult, err = UpdateStatus(context.Background(), c, binding, func() error { + binding.Status.AggregatedStatus = aggregatedStatuses + // set binding status with the newest condition + meta.SetStatusCondition(&binding.Status.Conditions, fullyAppliedCondition) return nil - } - - updated := &workv1alpha2.ResourceBinding{} - if err = c.Get(context.TODO(), client.ObjectKey{Namespace: binding.Namespace, Name: binding.Name}, updated); err == nil { - binding = updated - } else { - klog.Errorf("Failed to get updated binding %s/%s: %v", binding.Namespace, binding.Name, err) - } - - return updateErr - }) - if err != nil { + }) + return err + }); err != nil { eventRecorder.Event(binding, corev1.EventTypeWarning, events.EventReasonAggregateStatusFailed, err.Error()) eventRecorder.Event(resourceTemplate, corev1.EventTypeWarning, events.EventReasonAggregateStatusFailed, err.Error()) return err } - msg := fmt.Sprintf("Update resourceBinding(%s/%s) with AggregatedStatus successfully.", binding.Namespace, binding.Name) - eventRecorder.Event(binding, corev1.EventTypeNormal, events.EventReasonAggregateStatusSucceed, msg) - eventRecorder.Event(resourceTemplate, corev1.EventTypeNormal, events.EventReasonAggregateStatusSucceed, msg) + if operationResult == controllerutil.OperationResultUpdatedStatusOnly { + msg := fmt.Sprintf("Update ResourceBinding(%s/%s) with AggregatedStatus successfully.", binding.Namespace, binding.Name) + eventRecorder.Event(binding, corev1.EventTypeNormal, events.EventReasonAggregateStatusSucceed, msg) + eventRecorder.Event(resourceTemplate, corev1.EventTypeNormal, events.EventReasonAggregateStatusSucceed, msg) + } else { + klog.Infof("New aggregatedStatuses are equal with old ResourceBinding(%s/%s) AggregatedStatus, no update required.", binding.Namespace, binding.Name) + } return nil } @@ -131,40 +118,29 @@ func AggregateClusterResourceBindingWorkStatus( fullyAppliedCondition := generateFullyAppliedCondition(binding.Spec, aggregatedStatuses) - err = retry.RetryOnConflict(retry.DefaultRetry, func() (err error) { - currentBindingStatus := binding.Status.DeepCopy() - - binding.Status.AggregatedStatus = aggregatedStatuses - // set binding status with the newest condition - meta.SetStatusCondition(&binding.Status.Conditions, fullyAppliedCondition) - if reflect.DeepEqual(binding.Status, *currentBindingStatus) { - klog.Infof("New aggregatedStatuses are equal with old clusterResourceBinding(%s) AggregatedStatus, no update required.", binding.Name) + var operationResult controllerutil.OperationResult + if err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + operationResult, err = UpdateStatus(context.Background(), c, binding, func() error { + binding.Status.AggregatedStatus = aggregatedStatuses + // set binding status with the newest condition + meta.SetStatusCondition(&binding.Status.Conditions, fullyAppliedCondition) return nil - } - - updateErr := c.Status().Update(context.TODO(), binding) - if updateErr == nil { - return nil - } - - updated := &workv1alpha2.ClusterResourceBinding{} - if err = c.Get(context.TODO(), client.ObjectKey{Name: binding.Name}, updated); err == nil { - binding = updated - } else { - klog.Errorf("Failed to get updated binding %s/%s: %v", binding.Namespace, binding.Name, err) - } - - return updateErr - }) - if err != nil { + }) + return err + }); err != nil { eventRecorder.Event(binding, corev1.EventTypeWarning, events.EventReasonAggregateStatusFailed, err.Error()) eventRecorder.Event(resourceTemplate, corev1.EventTypeWarning, events.EventReasonAggregateStatusFailed, err.Error()) return err } - msg := fmt.Sprintf("Update clusterResourceBinding(%s) with AggregatedStatus successfully.", binding.Name) - eventRecorder.Event(binding, corev1.EventTypeNormal, events.EventReasonAggregateStatusSucceed, msg) - eventRecorder.Event(resourceTemplate, corev1.EventTypeNormal, events.EventReasonAggregateStatusSucceed, msg) + if operationResult == controllerutil.OperationResultUpdatedStatusOnly { + msg := fmt.Sprintf("Update ClusterResourceBinding(%s) with AggregatedStatus successfully.", binding.Name) + eventRecorder.Event(binding, corev1.EventTypeNormal, events.EventReasonAggregateStatusSucceed, msg) + eventRecorder.Event(resourceTemplate, corev1.EventTypeNormal, events.EventReasonAggregateStatusSucceed, msg) + } else { + klog.Infof("New aggregatedStatuses are equal with old ClusterResourceBinding(%s) AggregatedStatus, no update required.", binding.Name) + } + return nil } diff --git a/pkg/util/lifted/discovery_test.go b/pkg/util/lifted/discovery_test.go index e7c9d1ff4ee2..4637b2643aa5 100644 --- a/pkg/util/lifted/discovery_test.go +++ b/pkg/util/lifted/discovery_test.go @@ -1,9 +1,12 @@ /* Copyright 2016 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/util/lifted/validatingmcs.go b/pkg/util/lifted/validatingmcs.go index 6e61c7bb38e2..cb573e89d6b1 100644 --- a/pkg/util/lifted/validatingmcs.go +++ b/pkg/util/lifted/validatingmcs.go @@ -22,7 +22,7 @@ package lifted import ( corev1 "k8s.io/api/core/v1" - validation "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" netutils "k8s.io/utils/net" ) diff --git a/pkg/util/proxy/upgrader.go b/pkg/util/proxy/upgrader.go index 3c5242a17a02..b0a314471060 100644 --- a/pkg/util/proxy/upgrader.go +++ b/pkg/util/proxy/upgrader.go @@ -292,7 +292,7 @@ func (h *UpgradeAwareHandler) tryUpgrade(w http.ResponseWriter, req *http.Reques if len(rawResponse) > 0 { klog.V(6).Infof("Writing %d bytes to hijacked connection", len(rawResponse)) if _, err = requestHijackedConn.Write(rawResponse); err != nil { - utilruntime.HandleError(fmt.Errorf("Error proxying response from backend to client: %v", err)) + utilruntime.HandleError(fmt.Errorf("error proxying response from backend to client: %v", err)) } } diff --git a/pkg/webhook/cronfederatedhpa/validating.go b/pkg/webhook/cronfederatedhpa/validating.go index 507d63afa0d9..e1b6385e265e 100755 --- a/pkg/webhook/cronfederatedhpa/validating.go +++ b/pkg/webhook/cronfederatedhpa/validating.go @@ -1,9 +1,12 @@ /* Copyright 2023 The Karmada Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/webhook/resourcedeletionprotection/validating.go b/pkg/webhook/resourcedeletionprotection/validating.go index 020e312b478c..d2fb0b52b1bd 100755 --- a/pkg/webhook/resourcedeletionprotection/validating.go +++ b/pkg/webhook/resourcedeletionprotection/validating.go @@ -1,9 +1,12 @@ /* Copyright 2023 The Karmada Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/test/e2e/cronfederatedhpa_test.go b/test/e2e/cronfederatedhpa_test.go index b021ff108f91..ecef1c8d421f 100644 --- a/test/e2e/cronfederatedhpa_test.go +++ b/test/e2e/cronfederatedhpa_test.go @@ -1,9 +1,12 @@ /* Copyright 2023 The Karmada Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/test/e2e/federatedhpa_test.go b/test/e2e/federatedhpa_test.go index f80f0d1d2d1c..87de12088d42 100644 --- a/test/e2e/federatedhpa_test.go +++ b/test/e2e/federatedhpa_test.go @@ -1,9 +1,12 @@ /* Copyright 2024 The Karmada Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/test/e2e/framework/cluster.go b/test/e2e/framework/cluster.go index 272c296b1164..3215eeefa21d 100644 --- a/test/e2e/framework/cluster.go +++ b/test/e2e/framework/cluster.go @@ -25,6 +25,7 @@ import ( "github.com/onsi/gomega" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/dynamic" @@ -352,3 +353,20 @@ func SetClusterRegion(c client.Client, clusterName string, regionName string) er return true, nil }) } + +// UpdateClusterStatusCondition updates the target cluster status condition. +func UpdateClusterStatusCondition(client karmada.Interface, clusterName string, condition metav1.Condition) { + gomega.Eventually(func() (bool, error) { + cluster, err := client.ClusterV1alpha1().Clusters().Get(context.TODO(), clusterName, metav1.GetOptions{}) + if err != nil { + return false, err + } + + meta.SetStatusCondition(&cluster.Status.Conditions, condition) + _, err = client.ClusterV1alpha1().Clusters().UpdateStatus(context.TODO(), cluster, metav1.UpdateOptions{}) + if err != nil { + return false, err + } + return true, nil + }, pollTimeout, pollInterval).Should(gomega.Equal(true)) +} diff --git a/test/e2e/framework/cronfederatedhpa.go b/test/e2e/framework/cronfederatedhpa.go index ee0782951d5e..6516c9218def 100644 --- a/test/e2e/framework/cronfederatedhpa.go +++ b/test/e2e/framework/cronfederatedhpa.go @@ -1,9 +1,12 @@ /* Copyright 2023 The Karmada Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/test/e2e/framework/federatedhpa.go b/test/e2e/framework/federatedhpa.go index cfbffe518083..698c33341c7c 100644 --- a/test/e2e/framework/federatedhpa.go +++ b/test/e2e/framework/federatedhpa.go @@ -1,9 +1,12 @@ /* Copyright 2023 The Karmada Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/test/e2e/framework/karmadactl.go b/test/e2e/framework/karmadactl.go index 30c7b8e7a4cc..627ca88c25fc 100644 --- a/test/e2e/framework/karmadactl.go +++ b/test/e2e/framework/karmadactl.go @@ -64,7 +64,7 @@ func NewTestKubeconfig(kubeConfig, karmadaContext, karmadactlpath, namespace str // KarmadactlCmd runs the karmadactl executable through the wrapper script. func (tk *TestKubeconfig) KarmadactlCmd(args ...string) *exec.Cmd { - defaultArgs := []string{} + var defaultArgs []string if tk.KubeConfig != "" { defaultArgs = append(defaultArgs, "--"+clientcmd.RecommendedConfigPathFlag+"="+tk.KubeConfig) @@ -116,7 +116,7 @@ func (k *KarmadactlBuilder) exec() (string, error) { } // execWithFullOutput runs the karmadactl executable, and returns the stdout and stderr. -func (k KarmadactlBuilder) execWithFullOutput() (string, string, error) { +func (k *KarmadactlBuilder) execWithFullOutput() (string, string, error) { var stdout, stderr bytes.Buffer cmd := k.cmd cmd.Stdout, cmd.Stderr = &stdout, &stderr @@ -133,7 +133,7 @@ func (k KarmadactlBuilder) execWithFullOutput() (string, string, error) { if err != nil { var rc = 127 if ee, ok := err.(*exec.ExitError); ok { - rc = int(ee.Sys().(syscall.WaitStatus).ExitStatus()) + rc = ee.Sys().(syscall.WaitStatus).ExitStatus() } return stdout.String(), stderr.String(), uexec.CodeExitError{ Err: fmt.Errorf("error running %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v", cmd, cmd.Stdout, cmd.Stderr, err), diff --git a/test/e2e/lazy_activation_policy_test.go b/test/e2e/lazy_activation_policy_test.go index 637fa7e4b45f..78faff647396 100644 --- a/test/e2e/lazy_activation_policy_test.go +++ b/test/e2e/lazy_activation_policy_test.go @@ -158,6 +158,25 @@ var _ = ginkgo.Describe("Lazy activation policy testing", func() { }) }) + // Combined Case 3 (Policy preemption) + // refer: https://github.com/karmada-io/karmada/blob/release-1.9/docs/proposals/scheduling/activation-preference/lazy-activation-preference.md#combined-case-3-policy-preemption + + ginkgo.It("Combined Case 3 (Policy preemption)", func() { + ginkgo.By("step 1: deployment propagate success when policy created before it", func() { + waitDeploymentPresentOnCluster(originalCluster, namespace, deploymentName) + }) + + ginkgo.By("step 2: create PP2 (match nginx, cluster=member2, not lazy, priority=2, preemption=true)", func() { + policyHigherPriority.Spec.ActivationPreference = "" // remove lazy activationPreference field + framework.CreatePropagationPolicy(karmadaClient, policyHigherPriority) + waitDeploymentPresentOnCluster(modifiedCluster, namespace, deploymentName) + }) + + ginkgo.By("step 3: clean up", func() { + framework.RemovePropagationPolicyIfExist(karmadaClient, namespace, policyHigherPriorityName) + }) + }) + // Combined Case 4 (Policy preemption) // refer: https://github.com/karmada-io/karmada/blob/release-1.9/docs/proposals/scheduling/activation-preference/lazy-activation-preference.md#combined-case-4-policy-preemption ginkgo.It("Policy preemption", func() { diff --git a/test/e2e/remedy_test.go b/test/e2e/remedy_test.go index acaf9c66e693..00c321e56e40 100644 --- a/test/e2e/remedy_test.go +++ b/test/e2e/remedy_test.go @@ -17,12 +17,9 @@ limitations under the License. package e2e import ( - "context" "fmt" "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/sets" @@ -45,6 +42,9 @@ var _ = framework.SerialDescribe("remedy testing", func() { remedy = &remedyv1alpha1.Remedy{ ObjectMeta: metav1.ObjectMeta{Name: remedyName}, Spec: remedyv1alpha1.RemedySpec{ + ClusterAffinity: &remedyv1alpha1.ClusterAffinity{ + ClusterNames: []string{targetCluster}, + }, DecisionMatches: []remedyv1alpha1.DecisionMatch{ { ClusterConditionMatch: &remedyv1alpha1.ClusterConditionRequirement{ @@ -63,39 +63,31 @@ var _ = framework.SerialDescribe("remedy testing", func() { ginkgo.It("Cluster domain name resolution function encounters an exception and recover", func() { ginkgo.By(fmt.Sprintf("update Cluster(%s) %s condition to false", targetCluster, remedyv1alpha1.ServiceDomainNameResolutionReady), func() { - clusterObj, err := karmadaClient.ClusterV1alpha1().Clusters().Get(context.TODO(), targetCluster, metav1.GetOptions{}) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - - meta.SetStatusCondition(&clusterObj.Status.Conditions, metav1.Condition{ + framework.UpdateClusterStatusCondition(karmadaClient, targetCluster, metav1.Condition{ Type: string(remedyv1alpha1.ServiceDomainNameResolutionReady), Status: metav1.ConditionFalse, }) - _, err = karmadaClient.ClusterV1alpha1().Clusters().UpdateStatus(context.TODO(), clusterObj, metav1.UpdateOptions{}) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) }) - ginkgo.By("wait cluster status has TrafficControl RemedyAction", func() { + ginkgo.By(fmt.Sprintf("wait Cluster(%s) status has TrafficControl RemedyAction", targetCluster), func() { framework.WaitClusterFitWith(controlPlaneClient, targetCluster, func(cluster *clusterv1alpha1.Cluster) bool { actions := sets.NewString(cluster.Status.RemedyActions...) + fmt.Printf("Cluster(%s) remedyActions: %v\n", cluster.Name, actions) return actions.Has(string(remedyv1alpha1.TrafficControl)) }) }) ginkgo.By(fmt.Sprintf("recover Cluster(%s) %s condition to true", targetCluster, remedyv1alpha1.ServiceDomainNameResolutionReady), func() { - clusterObj, err := karmadaClient.ClusterV1alpha1().Clusters().Get(context.TODO(), targetCluster, metav1.GetOptions{}) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - - meta.SetStatusCondition(&clusterObj.Status.Conditions, metav1.Condition{ + framework.UpdateClusterStatusCondition(karmadaClient, targetCluster, metav1.Condition{ Type: string(remedyv1alpha1.ServiceDomainNameResolutionReady), Status: metav1.ConditionTrue, }) - _, err = karmadaClient.ClusterV1alpha1().Clusters().UpdateStatus(context.TODO(), clusterObj, metav1.UpdateOptions{}) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) }) - ginkgo.By("wait cluster status doesn't has TrafficControl RemedyAction", func() { + ginkgo.By(fmt.Sprintf("wait Cluster(%s) status doesn't has TrafficControl RemedyAction", targetCluster), func() { framework.WaitClusterFitWith(controlPlaneClient, targetCluster, func(cluster *clusterv1alpha1.Cluster) bool { actions := sets.NewString(cluster.Status.RemedyActions...) + fmt.Printf("Cluster(%s) remedyActions: %v\n", cluster.Name, actions) return !actions.Has(string(remedyv1alpha1.TrafficControl)) }) }) @@ -107,20 +99,16 @@ var _ = framework.SerialDescribe("remedy testing", func() { ginkgo.It("Cluster domain name resolution function encounters an exception, then remove the remedy resource", func() { ginkgo.By(fmt.Sprintf("update Cluster(%s) %s condition to false", targetCluster, remedyv1alpha1.ServiceDomainNameResolutionReady), func() { - clusterObj, err := karmadaClient.ClusterV1alpha1().Clusters().Get(context.TODO(), targetCluster, metav1.GetOptions{}) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - - meta.SetStatusCondition(&clusterObj.Status.Conditions, metav1.Condition{ + framework.UpdateClusterStatusCondition(karmadaClient, targetCluster, metav1.Condition{ Type: string(remedyv1alpha1.ServiceDomainNameResolutionReady), Status: metav1.ConditionFalse, }) - _, err = karmadaClient.ClusterV1alpha1().Clusters().UpdateStatus(context.TODO(), clusterObj, metav1.UpdateOptions{}) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) }) - ginkgo.By("wait cluster status has TrafficControl RemedyAction", func() { + ginkgo.By(fmt.Sprintf("wait Cluster(%s) status has TrafficControl RemedyAction", targetCluster), func() { framework.WaitClusterFitWith(controlPlaneClient, targetCluster, func(cluster *clusterv1alpha1.Cluster) bool { actions := sets.NewString(cluster.Status.RemedyActions...) + fmt.Printf("Cluster(%s) remedyActions: %v\n", cluster.Name, actions) return actions.Has(string(remedyv1alpha1.TrafficControl)) }) }) @@ -129,23 +117,19 @@ var _ = framework.SerialDescribe("remedy testing", func() { karmadaresource.RemoveRemedy(karmadaClient, remedyName) }) - ginkgo.By("wait cluster status doesn't has TrafficControl RemedyAction", func() { + ginkgo.By(fmt.Sprintf("wait Cluster(%s) status doesn't has TrafficControl RemedyAction", targetCluster), func() { framework.WaitClusterFitWith(controlPlaneClient, targetCluster, func(cluster *clusterv1alpha1.Cluster) bool { actions := sets.NewString(cluster.Status.RemedyActions...) + fmt.Printf("Cluster(%s) remedyActions: %v\n", cluster.Name, actions) return !actions.Has(string(remedyv1alpha1.TrafficControl)) }) }) ginkgo.By(fmt.Sprintf("cleanup: recover Cluster(%s) %s to true", targetCluster, remedyv1alpha1.ServiceDomainNameResolutionReady), func() { - clusterObj, err := karmadaClient.ClusterV1alpha1().Clusters().Get(context.TODO(), targetCluster, metav1.GetOptions{}) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - - meta.SetStatusCondition(&clusterObj.Status.Conditions, metav1.Condition{ + framework.UpdateClusterStatusCondition(karmadaClient, targetCluster, metav1.Condition{ Type: string(remedyv1alpha1.ServiceDomainNameResolutionReady), Status: metav1.ConditionTrue, }) - _, err = karmadaClient.ClusterV1alpha1().Clusters().UpdateStatus(context.TODO(), clusterObj, metav1.UpdateOptions{}) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) }) }) }) @@ -157,10 +141,12 @@ var _ = framework.SerialDescribe("remedy testing", func() { remedy = &remedyv1alpha1.Remedy{ ObjectMeta: metav1.ObjectMeta{Name: remedyName}, Spec: remedyv1alpha1.RemedySpec{ + ClusterAffinity: &remedyv1alpha1.ClusterAffinity{ + ClusterNames: []string{targetCluster}, + }, Actions: []remedyv1alpha1.RemedyAction{remedyv1alpha1.TrafficControl}, }, } - }) ginkgo.It("Create an immediately type remedy, then remove it", func() { @@ -168,9 +154,10 @@ var _ = framework.SerialDescribe("remedy testing", func() { karmadaresource.CreateRemedy(karmadaClient, remedy) }) - ginkgo.By("wait cluster status has TrafficControl RemedyAction", func() { + ginkgo.By(fmt.Sprintf("wait Cluster(%s) status has TrafficControl RemedyAction", targetCluster), func() { framework.WaitClusterFitWith(controlPlaneClient, targetCluster, func(cluster *clusterv1alpha1.Cluster) bool { actions := sets.NewString(cluster.Status.RemedyActions...) + fmt.Printf("Cluster(%s) remedyActions: %v\n", cluster.Name, actions) return actions.Has(string(remedyv1alpha1.TrafficControl)) }) }) @@ -179,9 +166,10 @@ var _ = framework.SerialDescribe("remedy testing", func() { karmadaresource.RemoveRemedy(karmadaClient, remedyName) }) - ginkgo.By("wait cluster status doesn't has TrafficControl RemedyAction", func() { + ginkgo.By(fmt.Sprintf("wait Cluster(%s) status doesn't has TrafficControl RemedyAction", targetCluster), func() { framework.WaitClusterFitWith(controlPlaneClient, targetCluster, func(cluster *clusterv1alpha1.Cluster) bool { actions := sets.NewString(cluster.Status.RemedyActions...) + fmt.Printf("Cluster(%s) remedyActions: %v\n", cluster.Name, actions) return !actions.Has(string(remedyv1alpha1.TrafficControl)) }) }) diff --git a/vendor/k8s.io/kube-openapi/cmd/openapi-gen/openapi-gen.go b/vendor/k8s.io/kube-openapi/cmd/openapi-gen/openapi-gen.go new file mode 100644 index 000000000000..c446e80b8a02 --- /dev/null +++ b/vendor/k8s.io/kube-openapi/cmd/openapi-gen/openapi-gen.go @@ -0,0 +1,57 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This package generates openAPI definition file to be used in open API spec generation on API servers. To generate +// definition for a specific type or package add "+k8s:openapi-gen=true" tag to the type/package comment lines. To +// exclude a type from a tagged package, add "+k8s:openapi-gen=false" tag to the type comment lines. + +package main + +import ( + "flag" + "log" + + generatorargs "k8s.io/kube-openapi/cmd/openapi-gen/args" + "k8s.io/kube-openapi/pkg/generators" + + "github.com/spf13/pflag" + + "k8s.io/klog/v2" +) + +func main() { + klog.InitFlags(nil) + genericArgs, customArgs := generatorargs.NewDefaults() + + genericArgs.AddFlags(pflag.CommandLine) + customArgs.AddFlags(pflag.CommandLine) + flag.Set("logtostderr", "true") + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + pflag.Parse() + + if err := generatorargs.Validate(genericArgs); err != nil { + log.Fatalf("Arguments validation error: %v", err) + } + + // Generates the code for the OpenAPIDefinitions. + if err := genericArgs.Execute( + generators.NameSystems(), + generators.DefaultNameSystem(), + generators.Packages, + ); err != nil { + log.Fatalf("OpenAPI code generation error: %v", err) + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 941cde40e81b..129a7a1dca6e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1643,6 +1643,7 @@ k8s.io/kube-aggregator/pkg/registry/apiservice/etcd k8s.io/kube-aggregator/pkg/registry/apiservice/rest # k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 ## explicit; go 1.19 +k8s.io/kube-openapi/cmd/openapi-gen k8s.io/kube-openapi/cmd/openapi-gen/args k8s.io/kube-openapi/pkg/aggregator k8s.io/kube-openapi/pkg/builder