From a99b5e389afdcf709c92ee2ac3e0559d91087fad Mon Sep 17 00:00:00 2001 From: shaofan-hs <133250733+shaofan-hs@users.noreply.github.com> Date: Tue, 22 Aug 2023 22:30:48 +0800 Subject: [PATCH] fix some bugs encountered by integration tests (#34) * fix, some bugs encountered by integration tests --- Dockerfile | 8 +- config/manager/manager.yaml | 20 +- config/rbac/kustomization.yaml | 2 + config/rbac/webhook_role.yaml | 40 ++++ config/rbac/webhook_role_binding.yaml | 19 ++ config/webhook/kustomization.yaml | 1 - config/webhook/secret.yaml | 8 - config/webhook/service.yaml | 2 +- config/webhook/webhook.yaml | 22 +- main.go | 19 +- .../podopslifecycle_controller.go | 103 +++++--- .../podopslifecycle_controller_test.go | 11 +- pkg/controllers/podopslifecycle/predicate.go | 17 +- pkg/controllers/ruleset/checker/checker.go | 3 + .../utils/expectations/expectation.go | 14 +- .../resourceversion_expectation.go | 12 +- pkg/{ => utils}/log/logger.go | 0 pkg/utils/pki_helpers.go | 87 +++++++ .../generic/generic_mutating_handler.go | 3 + .../generic/generic_validating_handler.go | 3 + .../generic/pod/opslifecycle/mutating.go | 110 +++++---- .../generic/pod/opslifecycle/webhook.go | 46 ++-- .../generic/pod/opslifecycle/webhook_test.go | 47 ++-- .../generic/pod/pod_validating_handler.go | 6 +- pkg/webhook/webhook.go | 220 ++++++++++++++++++ test/e2e/scripts/kind-conf.yaml | 2 +- 26 files changed, 637 insertions(+), 188 deletions(-) create mode 100644 config/rbac/webhook_role.yaml create mode 100644 config/rbac/webhook_role_binding.yaml delete mode 100644 config/webhook/secret.yaml rename pkg/{ => utils}/log/logger.go (100%) create mode 100644 pkg/utils/pki_helpers.go diff --git a/Dockerfile b/Dockerfile index e29adfa4..36cc6091 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.19 as builder +FROM golang:1.19-alpine3.17 as builder ARG TARGETOS ARG TARGETARCH @@ -23,11 +23,9 @@ COPY pkg/ pkg/ # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} go build -a -o manager main.go -# Use distroless as minimal base image to package the manager binary -# Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM gcr.io/distroless/static:nonroot +FROM alpine:3.17 +RUN mkdir /webhook-certs WORKDIR / COPY --from=builder /workspace/manager . -USER 65532:65532 ENTRYPOINT ["/manager"] diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index c3d66c0e..d7665fe6 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -56,15 +56,6 @@ spec: # operator: In # values: # - linux - securityContext: - runAsNonRoot: true - # TODO(user): For common cases that do not require escalating privileges - # it is recommended to ensure that all your Pods/Containers are restrictive. - # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted - # Please uncomment the following code if your project does NOT have to work on old Kubernetes - # versions < 1.19 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ). - # seccompProfile: - # type: RuntimeDefault containers: - command: - /manager @@ -72,9 +63,16 @@ spec: args: - "--leader-elect" - "--cert-dir=/webhook-certs" + - "--dns-name=kusionstack-controller-manager.kusionstack-system.svc" - "--health-probe-bind-address=:8081" - "--metrics-bind-address=127.0.0.1:8080" + - "-v=4" name: manager + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace securityContext: allowPrivilegeEscalation: false capabilities: @@ -101,10 +99,6 @@ spec: requests: cpu: 10m memory: 64Mi - volumeMounts: - - name: webhook-certs - mountPath: "/webhook-certs" - readOnly: true serviceAccountName: controller-manager terminationGracePeriodSeconds: 0 volumes: diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 731832a6..a5779c6d 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -9,6 +9,8 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml +- webhook_role.yaml +- webhook_role_binding.yaml # Comment the following 4 lines if you want to disable # the auth proxy (https://github.com/brancz/kube-rbac-proxy) # which protects your /metrics endpoint. diff --git a/config/rbac/webhook_role.yaml b/config/rbac/webhook_role.yaml new file mode 100644 index 00000000..1a9310a7 --- /dev/null +++ b/config/rbac/webhook_role.yaml @@ -0,0 +1,40 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: webhook-role +rules: +- apiGroups: + - "admissionregistration.k8s.io" + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - secrets/status + verbs: + - get + - patch + - update \ No newline at end of file diff --git a/config/rbac/webhook_role_binding.yaml b/config/rbac/webhook_role_binding.yaml new file mode 100644 index 00000000..6fbadcf1 --- /dev/null +++ b/config/rbac/webhook_role_binding.yaml @@ -0,0 +1,19 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: clusterrolebinding + app.kubernetes.io/instance: webhook-rolebinding + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: kusionstack + app.kubernetes.io/part-of: kusionstack + app.kubernetes.io/managed-by: kustomize + name: webhook-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: webhook-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml index 92afc12e..730d9267 100644 --- a/config/webhook/kustomization.yaml +++ b/config/webhook/kustomization.yaml @@ -1,4 +1,3 @@ resources: -- secret.yaml - service.yaml - webhook.yaml diff --git a/config/webhook/secret.yaml b/config/webhook/secret.yaml deleted file mode 100644 index 7e0e1abb..00000000 --- a/config/webhook/secret.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - namespace: kusionstack-system - name: webhook-certs -data: - tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURYekNDQWtlZ0F3SUJBZ0lJS0pLSktJZkwwd1V3RFFZSktvWklodmNOQVFFTEJRQXdIekVkTUJzR0ExVUUKQXhNVWMyVnNaaTF6YVdkdVpXUXRhemh6TFdObGNuUXdJQmNOTWpNd056RTNNRFl5T1RNeFdoZ1BNakV5TXpBMgpNak13TmpJNU16RmFNRGd4TmpBMEJnTlZCQU1UTFd0dmNtSnBkRzh0WTI5dWRISnZiR3hsY2kxdFlXNWhaMlZ5CkxtdHZjbUpwZEc4dGMzbHpkR1Z0TG5OMll6Q0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0MKZ2dFQkFOLzZDZUJmNVNqZ0dKMENwQ1hPbGJLMFJvTGpXOWJGMnhHdVNZSmVQQWVXeUJzNk85aXlJci9WbFZFWgpKSElPVWZQRXVHU3lRbm9wQUMrbEpxdE4rNHBOalkvNHFiZ1RHUW1FbW50R05ZaTdDSnBPRC9uTmhJUDNlMlVmCnhQNlNncXFkaUtFakRUdStLbWJzVVQycktwNGJjRDludFFTdHBmcFN2WkNQZlJRaFcvN3NsRVU2K0xwZkxmcjgKWTB3d25HUWV2bDAzWGJSY3ZQWTF2L0R3ejNpYU96d05kZHpBWkp0cWlPenYza3I1WEF2bTQ0cGllNXZGSFZKZQpzN2RKR2xibUd2ZlRqR0E2c0hmRXdPYjFnR0V4N1lBYjdHNlkrUVc5VlQ0M0lJSVZ2ZEZzSGR0a3JHRERoTTE3CjNEYk9Mdzk5cFIzZDRmeGlxMWlScXhTeEFBMENBd0VBQWFPQmd6Q0JnREFPQmdOVkhROEJBZjhFQkFNQ0JhQXcKRXdZRFZSMGxCQXd3Q2dZSUt3WUJCUVVIQXdFd0h3WURWUjBqQkJnd0ZvQVVzTERRWCtISm1kRk1TaWFWL29wMQpQMTRDVjhFd09BWURWUjBSQkRFd0w0SXRhMjl5WW1sMGJ5MWpiMjUwY205c2JHVnlMVzFoYm1GblpYSXVhMjl5ClltbDBieTF6ZVhOMFpXMHVjM1pqTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFCT3JEZWhYWkw2V0UxcVlsUU0KM1Y1YmNjVnhUaVFvTm9zelZ6WGdFY0xmdThrVUlVTFpTR1V0N3hRVE9teFg3TXMyMWxCRnd2VHlLL1lDYmdiRwpxZE9QV1Y2Q2Y0ZTd2MXlnSkVrOUlJVXd2c0YyTHFxVmZwa0l3S3psNEMrc1IwRWF0QlZOK283bGwxd2drWXRYCmV2OXEyayt3NW11cEZPQmpzRVIrSU5rQWFJZmdoWjlkb0FEWG5hbHpDeW5hVmlJWUw1SG5ReC9TTUt3bSt5cVIKNDhWWG1tZUVINCtBUHZtV0NySW55SFJrVTZSVzMxNENucU9Dc0Ziajl6WHdYa01vVk81VU1nYjZmNTI0MGZNMwpOeituRUNMTmhpWTJFbFdoM0xFR1IzUmF1RDBEQ1lydXNSZXVSMjFmK2JSUm9xUnI4ZGk0bTVFY240czdvVlBNClJtQ3EKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBMy9vSjRGL2xLT0FZblFLa0pjNlZzclJHZ3VOYjFzWGJFYTVKZ2w0OEI1YklHem83CjJMSWl2OVdWVVJra2NnNVI4OFM0WkxKQ2Vpa0FMNlVtcTAzN2lrMk5qL2lwdUJNWkNZU2FlMFkxaUxzSW1rNFAKK2MyRWcvZDdaUi9FL3BLQ3FwMklvU01OTzc0cVp1eFJQYXNxbmh0d1AyZTFCSzJsK2xLOWtJOTlGQ0ZiL3V5VQpSVHI0dWw4dCt2eGpURENjWkI2K1hUZGR0Rnk4OWpXLzhQRFBlSm83UEExMTNNQmttMnFJN08vZVN2bGNDK2JqCmltSjdtOFVkVWw2enQwa2FWdVlhOTlPTVlEcXdkOFRBNXZXQVlUSHRnQnZzYnBqNUJiMVZQamNnZ2hXOTBXd2QKMjJTc1lNT0V6WHZjTnM0dkQzMmxIZDNoL0dLcldKR3JGTEVBRFFJREFRQUJBb0lCQUVUQVJzbldCY3pQUlI3bApFYWY5U3R4SkZCL1hWSGlQMFZnVERNTjAwNjVzclU0V1NRcVNXN0pSQ0gyVlh5ZVhsN1poRzB6bG54eEtlNkVrCmE4TkhzNy9YWUczaUlZNjhaZ0lTN3l5Sjk2TFo3cUVXOXpaeHpJeUpxaG4zYjR2SlEwekdBSXVkNjlENzVaNGsKU2lmSTQ5TVdqeFVYYTJ4ZE5VYXFvVCsvbStCU2JoTG5mTFN0ZXlIKzNOdHhqMGRnZDVpb1ZLMW9DakY0bjB0SgpvQmJRRTBWL3RyRWtLeEhWSFdxOTNBR3Znc1NmTFRkOUJrc0ltbUFLUjJIL1JkOHNyazNoQk1wL0lwWms0Nm0wCm0zaURrZEp1UzlHc0tvUjc3QlM4OFdHUUpCR3RUcnB3UTdHTEdSUUcwOFpvdjA3SHYvU1NIYXRsSlBjbGFBNmYKN3QwaUh3RUNnWUVBOVdvbFlIYkVZSDh3cVo5Z0gyektmNDlxWnNTMzBrVTJaVW05elowRWozRGxTSG44SnA2YgpaTEZTb3d2MEYxcXl3U013RXN5SllHQjRSaU56M2lmYmJobHB2SWJzTDNja1VEdzFMQXZ1R2loRE9BbkZqQ0VRCnVkYUxNNFdYLzBrelV2NzNYVGhCMmZhejNUWER0YzBkYVh3dXBMaFhuOHpXSTBEbm10Tmp1TTBDZ1lFQTZhTXQKSTVlUUZoczNGVXhMMTlpaDBqNnJ6STlEV0Z3cVpHYUJKSEs5OTFqZTRHV2VSSkRsZnZmYjMrM1hxejcvTGJwZwpsdjRHUFJ5U2hUT2trLzRWTXR0UkYrOVZSY0dId29ldWc5cEtVWmdoZmt3Z2ZtYzdkQ0NDbm8xc3B2VVhNcDNXCkF5RjNHNng2SG9LTTBya1BIMVpnc3lJYldMZ1FZT1czZFJEclpFRUNnWUVBejlvUXdmeVZJZTFYZDdJbUpUZHMKNGxEREd6c3RQUWVscEcrcU00VHpJZEhLUDNJaFgwd2g3RXhLNHhEeXNQeitQSm5pMDJrc1NOWXJ4NnVyTE5LeApCRFM5MnhDaTVTaVJOdXRuVldEZFRvWWZPRXorWlVwejhoR0ZZQ09ic25sNjRLWlZOenk0ZVRSNjdiWGxoZ3F0CkNIUkd4UGZrTHRHNzQ2dksvTlJndDFrQ2dZQjZrVFVqLytZQ0F1TUIxRlhSZFg5bk9hT05HL2M3aXBaZXQzdTIKY0UrQ1RHZ1lGcGRSNXlYT1Z4OU5PWXd6OVVlUGFNQTFWaXhWc1FDWXluL2pkNThOWUFzSjRHKyt1WW1NSHZ3SgpoZUlKTGlNY0M3bmNOakgxZVB5dFN3b2VDR1BVRlFOcC93dHFSRENJZ3ZwM3JjZm1LNzF0SEU1Mlc1R1VFODhMCnBnV25RUUtCZ0ZVa2dnTGx4UVgwcnQrUEEzZnBBZHNGUXYxTElKNUR1Q05DRWIrK1RQL3FVM3FYclBsRUdZeVkKUFB6U3FXNkZLa2ZEVTFRdjJlN3BMSzdra2RpZld0bHNyNkdDVGlEN01kcm5Wc3RuM2xEdC90ck56a3NoS00xUwplY0xLWGp2N1MveXFEbzBETm1MVml2TDhWU0tLdVpKUGN5VktOeGZIOHJYV3BIaGQ4MkQ1Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== \ No newline at end of file diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml index 1087a23a..22b47ffe 100644 --- a/config/webhook/service.yaml +++ b/config/webhook/service.yaml @@ -8,4 +8,4 @@ spec: - port: 443 targetPort: 9443 selector: - control-panel: controller-manager + control-plane: controller-manager diff --git a/config/webhook/webhook.yaml b/config/webhook/webhook.yaml index a7141515..72d9131a 100644 --- a/config/webhook/webhook.yaml +++ b/config/webhook/webhook.yaml @@ -10,8 +10,8 @@ webhooks: service: namespace: kusionstack-system name: controller-manager - caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMvVENDQWVXZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFmTVIwd0d3WURWUVFERXhSelpXeG0KTFhOcFoyNWxaQzFyT0hNdFkyVnlkREFnRncweU16QTNNVGN3TmpJNU16RmFHQTh5TVRJek1EWXlNekEyTWpregpNVm93SHpFZE1Cc0dBMVVFQXhNVWMyVnNaaTF6YVdkdVpXUXRhemh6TFdObGNuUXdnZ0VpTUEwR0NTcUdTSWIzCkRRRUJBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRRHhEQytBS05oSGVxRThqaThBNllIUGNZbGVyeERrMmNoQlpabzAKaEJqTUpMZno1STU0aENhR0wwT08vMU0yMFQyZnFZWEFrRWwzRnlhU1VIY3liNnNGbEMwWHRkLzVaK0tMZkRKTgpTK2YrdHB3QmxZZ3U0S2hHN1U5VmpiV3RZRWk2OGZKNFNIRHBGd3BWZnFzSzhhVjYrZis1cElPclZFYS9rbmhsCitFd2ZBeG1uNm1xVlpZQXhManBVNXF3TERqU3ZXcnhIcTQ2UWx1eTBwV09maXBYelg4L3BLT0d2YWN6L1R2emMKRy9uNnY1NDNSeXArV05PV0hvajdSTXA3YTVYczdQcjFMM040ZjhscWJkMWs3WGZJa1lXWlR2OWpqeFRFRFp6Wgo3Y3BwRXZ1OHRBK0MxVzhMeDdOSGk1a1BXcjM5YUhkb201NUpHT2tWZDdEdDB2RHhBZ01CQUFHalFqQkFNQTRHCkExVWREd0VCL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlN3c05CZjRjbVoKMFV4S0pwWCtpblUvWGdKWHdUQU5CZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFXWFNibXlMWXAwZHFXTjVaaHNXVgphWUwxbEh4SmlyaE5IbHZqYkM2cXpnd2VUNWRJWFB6U3lQZ25DajBDOHJ1bHJiQUV4R3Jva1hkQzJiVTBoYUw3CngxU2M4R1lPSU9pSFdHQnM1VitrbUh0bzdmeVR4cFV0OGFSNU1TWitCZkFBNHJRZzJWRWNxUkkzRE9aTDdRYXAKRVZLWnpqSTJObkRRbUN2N2oxZERrajVRMWRsTW96QWRlN1ZUZXE0Y1pVTW8ydUNmeEViZlZMSXVzRXI3cmc1cwpIa2M4U3piVUpudTdDc0dRbE1JNTBMV3FxWHlkT3ZCNk5nTjhvNDNBdlY0ck9NOGx5WklnbG14ZkRPMVRyUlVCCmMza0wzZ0JHazlHZnJ0OFI1d1dOamlMQkJRQnBRMFdqVlh2QlNaSzdOcHlYSEFranVUSjlTa0ZaQXBUZ2JmM2UKSFE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - failurePolicy: Ignore + path: /mutating-generic + failurePolicy: Fail rules: - apiGroups: - "*" @@ -20,10 +20,16 @@ webhooks: operations: - CREATE - UPDATE + - DELETE resources: - pods scope: '*' - + objectSelector: + matchExpressions: + - key: podopslifecycle.kusionstack.io/control + operator: In + values: + - 'true' --- apiVersion: admissionregistration.k8s.io/v1 @@ -38,8 +44,8 @@ webhooks: service: namespace: kusionstack-system name: controller-manager - caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMvVENDQWVXZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFmTVIwd0d3WURWUVFERXhSelpXeG0KTFhOcFoyNWxaQzFyT0hNdFkyVnlkREFnRncweU16QTNNVGN3TmpJNU16RmFHQTh5TVRJek1EWXlNekEyTWpregpNVm93SHpFZE1Cc0dBMVVFQXhNVWMyVnNaaTF6YVdkdVpXUXRhemh6TFdObGNuUXdnZ0VpTUEwR0NTcUdTSWIzCkRRRUJBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRRHhEQytBS05oSGVxRThqaThBNllIUGNZbGVyeERrMmNoQlpabzAKaEJqTUpMZno1STU0aENhR0wwT08vMU0yMFQyZnFZWEFrRWwzRnlhU1VIY3liNnNGbEMwWHRkLzVaK0tMZkRKTgpTK2YrdHB3QmxZZ3U0S2hHN1U5VmpiV3RZRWk2OGZKNFNIRHBGd3BWZnFzSzhhVjYrZis1cElPclZFYS9rbmhsCitFd2ZBeG1uNm1xVlpZQXhManBVNXF3TERqU3ZXcnhIcTQ2UWx1eTBwV09maXBYelg4L3BLT0d2YWN6L1R2emMKRy9uNnY1NDNSeXArV05PV0hvajdSTXA3YTVYczdQcjFMM040ZjhscWJkMWs3WGZJa1lXWlR2OWpqeFRFRFp6Wgo3Y3BwRXZ1OHRBK0MxVzhMeDdOSGk1a1BXcjM5YUhkb201NUpHT2tWZDdEdDB2RHhBZ01CQUFHalFqQkFNQTRHCkExVWREd0VCL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlN3c05CZjRjbVoKMFV4S0pwWCtpblUvWGdKWHdUQU5CZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFXWFNibXlMWXAwZHFXTjVaaHNXVgphWUwxbEh4SmlyaE5IbHZqYkM2cXpnd2VUNWRJWFB6U3lQZ25DajBDOHJ1bHJiQUV4R3Jva1hkQzJiVTBoYUw3CngxU2M4R1lPSU9pSFdHQnM1VitrbUh0bzdmeVR4cFV0OGFSNU1TWitCZkFBNHJRZzJWRWNxUkkzRE9aTDdRYXAKRVZLWnpqSTJObkRRbUN2N2oxZERrajVRMWRsTW96QWRlN1ZUZXE0Y1pVTW8ydUNmeEViZlZMSXVzRXI3cmc1cwpIa2M4U3piVUpudTdDc0dRbE1JNTBMV3FxWHlkT3ZCNk5nTjhvNDNBdlY0ck9NOGx5WklnbG14ZkRPMVRyUlVCCmMza0wzZ0JHazlHZnJ0OFI1d1dOamlMQkJRQnBRMFdqVlh2QlNaSzdOcHlYSEFranVUSjlTa0ZaQXBUZ2JmM2UKSFE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - failurePolicy: Ignore + path: /validating-generic + failurePolicy: Fail rules: - apiGroups: - "*" @@ -51,3 +57,9 @@ webhooks: resources: - pods scope: '*' + objectSelector: + matchExpressions: + - key: podopslifecycle.kusionstack.io/control + operator: In + values: + - 'true' \ No newline at end of file diff --git a/main.go b/main.go index bbe3227b..9868a6a4 100644 --- a/main.go +++ b/main.go @@ -17,6 +17,7 @@ limitations under the License. package main import ( + "context" "flag" "os" @@ -26,6 +27,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth" + "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -59,6 +61,7 @@ func main() { enableLeaderElection bool probeAddr string certDir string + dnsName string ) flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") @@ -66,6 +69,10 @@ func main() { "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") flag.StringVar(&certDir, "cert-dir", "", "The directory that contains the server key and certificate.") + flag.StringVar(&dnsName, "dns-name", "", "The DNS name of the webhook server.") + + klog.InitFlags(nil) + defer klog.Flush() opts := zap.Options{ Development: true, @@ -77,13 +84,14 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + config := ctrl.GetConfigOrDie() + mgr, err := ctrl.NewManager(config, ctrl.Options{ Scheme: scheme, MetricsBindAddress: metricsAddr, Port: 9443, HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, - LeaderElectionID: "5d84702b.kafed.io", + LeaderElectionID: "kusionstack-controller-manager", CertDir: certDir, NewCache: inject.NewCacheWithFieldIndex, @@ -119,6 +127,13 @@ func main() { os.Exit(1) } + // +kubebuilder:scaffold:builder + setupLog.Info("initialize webhook") + if err := webhook.Initialize(context.Background(), config, dnsName, certDir); err != nil { + setupLog.Error(err, "unable to initialize webhook") + os.Exit(1) + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/pkg/controllers/podopslifecycle/podopslifecycle_controller.go b/pkg/controllers/podopslifecycle/podopslifecycle_controller.go index 345672fd..862eade0 100644 --- a/pkg/controllers/podopslifecycle/podopslifecycle_controller.go +++ b/pkg/controllers/podopslifecycle/podopslifecycle_controller.go @@ -28,19 +28,19 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" - logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" "kusionstack.io/kafed/apis/apps/v1alpha1" "kusionstack.io/kafed/pkg/controllers/ruleset" - "kusionstack.io/kafed/pkg/controllers/utils" + controllerutils "kusionstack.io/kafed/pkg/controllers/utils" "kusionstack.io/kafed/pkg/controllers/utils/expectations" - "kusionstack.io/kafed/pkg/log" + "kusionstack.io/kafed/pkg/utils" ) const ( @@ -66,27 +66,31 @@ func AddToMgr(mgr manager.Manager, r reconcile.Reconciler) error { err = c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForObject{}, &PodPredicate{ NeedOpsLifecycle: func(oldPod, newPod *corev1.Pod) bool { - return true + return utils.ControlledByPodOpsLifecycle(newPod) }, }) if err != nil { return err } + sourceChannel := ruleset.RegisterListenChan(context.Background()) + err = c.Watch(sourceChannel, &handler.EnqueueRequestForObject{}) + if err != nil { + return err + } + return nil } var _ reconcile.Reconciler = &ReconcilePodOpsLifecycle{} func NewReconciler(mgr manager.Manager) *ReconcilePodOpsLifecycle { - logger := log.New(logf.Log.WithName("podopslifecycle-controller")) - expectation = expectations.NewResourceVersionExpectation(logger) + expectation = expectations.NewResourceVersionExpectation() r := &ReconcilePodOpsLifecycle{ Client: mgr.GetClient(), ruleSetManager: ruleset.RuleSetManager(), - logger: logger, recorder: mgr.GetEventRecorderFor(controllerName), expectation: expectation, } @@ -98,7 +102,6 @@ func NewReconciler(mgr manager.Manager) *ReconcilePodOpsLifecycle { type ReconcilePodOpsLifecycle struct { client.Client ruleSetManager ruleset.ManagerInterface - logger *log.Logger recorder record.EventRecorder expectation *expectations.ResourceVersionExpectation } @@ -107,12 +110,12 @@ type ReconcilePodOpsLifecycle struct { // +kubebuilder:rbac:groups=core,resources=pods/status,verbs=get;update;patch func (r *ReconcilePodOpsLifecycle) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { key := fmt.Sprintf("%s/%s", request.Namespace, request.Name) - r.logger.V(0).Infof("Reconcile Pod %s", key) + klog.Infof("Reconcile Pod %s", key) pod := &corev1.Pod{} err := r.Client.Get(ctx, request.NamespacedName, pod) if err != nil { - r.logger.Warningf("failed to get pod %s: %s", key, err) + klog.Warningf("failed to get pod %s: %s", key, err) if errors.IsNotFound(err) { r.expectation.DeleteExpectations(key) return reconcile.Result{}, nil @@ -122,15 +125,22 @@ func (r *ReconcilePodOpsLifecycle) Reconcile(ctx context.Context, request reconc state, err := r.ruleSetManager.GetState(r.Client, pod) if err != nil { - r.logger.Errorf("failed to get pod %s state: %s", key, err) + klog.Errorf("failed to get pod %s state: %s", key, err) return reconcile.Result{}, err } - var labels map[string]string + + var ( + stage string + labels map[string]string + ) if state.InStageAndPassed(v1alpha1.PodOpsLifecyclePreCheckStage) { + stage = v1alpha1.PodOpsLifecyclePreCheckStage labels, err = r.preCheckStage(pod) } else if state.InStageAndPassed(v1alpha1.PodOpsLifecyclePostCheckStage) { + stage = v1alpha1.PodOpsLifecyclePostCheckStage labels, err = r.postCheckStage(pod) } + klog.Infof("pod %s in stage %q, labels: %v, error: %v", key, stage, labels, err) if err != nil { return reconcile.Result{}, err } @@ -139,14 +149,13 @@ func (r *ReconcilePodOpsLifecycle) Reconcile(ctx context.Context, request reconc expectation.ExpectUpdate(key, pod.ResourceVersion) err = r.addLabels(ctx, pod, labels) if err != nil { - r.logger.Errorf("failed to update pod %s: %s", key, err) expectation.DeleteExpectations(key) } return reconcile.Result{}, err } if !r.expectation.SatisfiedExpectations(key, pod.ResourceVersion) { - r.logger.V(2).Infof("skip pod %s with no satisfied", key) + klog.Errorf("skip pod %s with no satisfied", key) return reconcile.Result{}, nil } @@ -156,8 +165,8 @@ func (r *ReconcilePodOpsLifecycle) Reconcile(ctx context.Context, request reconc } expected := map[string]bool{ - v1alpha1.PodPrepareLabelPrefix: false, // set readiness gate to false - v1alpha1.PodCompleteLabelPrefix: true, // set readiness gate to true + v1alpha1.PodPrepareLabelPrefix: false, // set readiness gate to false, traffic off + v1alpha1.PodCompleteLabelPrefix: true, // set readiness gate to true, traffic on } for _, labels := range idToLabelsMap { for k, v := range expected { @@ -165,25 +174,38 @@ func (r *ReconcilePodOpsLifecycle) Reconcile(ctx context.Context, request reconc continue } - needUpdate, _ := r.setServiceReadiness(pod, v) - if needUpdate { - r.expectation.ExpectUpdate(key, pod.ResourceVersion) - - if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - return r.Client.Status().Update(ctx, pod) - }); err != nil { - r.logger.Errorf("failed to update pod status %s: %s", key, err) - r.expectation.DeleteExpectations(key) - - return reconcile.Result{}, err - } - break + updated, err := r.updateServiceReadiness(ctx, pod, v) + if err != nil { + return reconcile.Result{}, err // only need set once + } + if updated { + return reconcile.Result{}, nil } - return reconcile.Result{}, nil // only need set once } } - return reconcile.Result{}, nil + _, err = r.updateServiceReadiness(ctx, pod, true) + return reconcile.Result{}, err +} + +func (r *ReconcilePodOpsLifecycle) updateServiceReadiness(ctx context.Context, pod *corev1.Pod, isReady bool) (bool, error) { + needUpdate, _ := r.setServiceReadiness(pod, isReady) + if !needUpdate { + return false, nil + } + + key := controllerKey(pod) + r.expectation.ExpectUpdate(key, pod.ResourceVersion) + + if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + return r.Client.Status().Update(ctx, pod) + }); err != nil { + klog.Errorf("failed to update pod status %s: %s", key, err) + r.expectation.DeleteExpectations(key) + + return false, err + } + return true, nil } func (r *ReconcilePodOpsLifecycle) setServiceReadiness(pod *corev1.Pod, isReady bool) (bool, string) { @@ -216,6 +238,7 @@ func (r *ReconcilePodOpsLifecycle) setServiceReadiness(pod *corev1.Pod, isReady Type: v1alpha1.ReadinessGatePodServiceReady, Status: status, LastTransitionTime: metav1.Now(), + Message: "updated by PodOpsLifecycle", }) return true, fmt.Sprintf("append service readiness gate to: %s", string(status)) } @@ -227,6 +250,8 @@ func (r *ReconcilePodOpsLifecycle) setServiceReadiness(pod *corev1.Pod, isReady // update readiness gate pod.Status.Conditions[index].Status = status pod.Status.Conditions[index].LastTransitionTime = metav1.Now() + pod.Status.Conditions[index].Message = "updated by PodOpsLifecycle" + return true, fmt.Sprintf("update service readiness gate to: %s", string(status)) } @@ -245,13 +270,13 @@ func (r *ReconcilePodOpsLifecycle) preCheckStage(pod *corev1.Pod) (labels map[st } key := fmt.Sprintf("%s/%s", v1alpha1.PodOperationPermissionLabelPrefix, t) - if _, ok := pod.GetLabels()[key]; !ok { - labels[key] = currentTime + if _, ok := pod.Labels[key]; !ok { + labels[key] = currentTime // operation-permission } key = fmt.Sprintf("%s/%s", v1alpha1.PodPreCheckedLabelPrefix, k) - if _, ok := pod.GetLabels()[key]; !ok { - labels[key] = currentTime + if _, ok := pod.Labels[key]; !ok { + labels[key] = currentTime // pre-checked } } @@ -269,7 +294,7 @@ func (r *ReconcilePodOpsLifecycle) postCheckStage(pod *corev1.Pod) (labels map[s for k := range idToLabelsMap { key := fmt.Sprintf("%s/%s", v1alpha1.PodPostCheckedLabelPrefix, k) if _, ok := pod.GetLabels()[key]; !ok { - labels[key] = currentTime + labels[key] = currentTime // post-checked } } @@ -299,10 +324,14 @@ func (r *ReconcilePodOpsLifecycle) initRuleSetManager() { return labels != nil && labelHasPrefix(labels, v1alpha1.PodPostCheckLabelPrefix) }) ruleset.AddUnAvailableFunc(func(po *corev1.Pod) (bool, *int64) { - return !utils.IsServiceAvailable(po), nil + return !controllerutils.IsServiceAvailable(po), nil }) } +func controllerKey(pod *corev1.Pod) string { + return fmt.Sprintf("%s/%s", pod.Namespace, pod.Name) +} + func labelHasPrefix(labels map[string]string, prefix string) bool { for k := range labels { if strings.HasPrefix(k, prefix) { diff --git a/pkg/controllers/podopslifecycle/podopslifecycle_controller_test.go b/pkg/controllers/podopslifecycle/podopslifecycle_controller_test.go index c96c8f0a..e1ae1643 100644 --- a/pkg/controllers/podopslifecycle/podopslifecycle_controller_test.go +++ b/pkg/controllers/podopslifecycle/podopslifecycle_controller_test.go @@ -138,6 +138,7 @@ var _ = Describe("podopslifecycle controller", func() { ObjectMeta: metav1.ObjectMeta{ Name: "test", Namespace: "default", + Labels: map[string]string{v1alpha1.ControlledByPodOpsLifecycle: "true"}, }, Spec: podSpec, } @@ -152,7 +153,9 @@ var _ = Describe("podopslifecycle controller", func() { Namespace: "default", }, pod) Expect(err).NotTo(HaveOccurred()) - Expect(pod.Status.Conditions).To(HaveLen(0)) + Expect(pod.Status.Conditions).To(HaveLen(1)) + Expect(string(pod.Status.Conditions[0].Type)).To(Equal(v1alpha1.ReadinessGatePodServiceReady)) + Expect(pod.Status.Conditions[0].Status).To(Equal(corev1.ConditionTrue)) podOpsLifecycle.ruleSetManager = &mockRuleSetManager{CheckState: &checker.CheckState{ States: []checker.State{ @@ -166,6 +169,7 @@ var _ = Describe("podopslifecycle controller", func() { }} pod.ObjectMeta.Labels = map[string]string{ + v1alpha1.ControlledByPodOpsLifecycle: "true", fmt.Sprintf("%s/%s", v1alpha1.PodOperateLabelPrefix, id): time, fmt.Sprintf("%s/%s", v1alpha1.PodPreCheckLabelPrefix, id): time, fmt.Sprintf("%s/%s", v1alpha1.PodOperationTypeLabelPrefix, id): operationType, @@ -192,6 +196,7 @@ var _ = Describe("podopslifecycle controller", func() { Name: "test", Namespace: "default", Labels: map[string]string{ + v1alpha1.ControlledByPodOpsLifecycle: "true", fmt.Sprintf("%s/%s", v1alpha1.PodOperatingLabelPrefix, id): time, fmt.Sprintf("%s/%s", v1alpha1.PodPrepareLabelPrefix, id): time, }, @@ -220,6 +225,7 @@ var _ = Describe("podopslifecycle controller", func() { Name: "test", Namespace: "default", Labels: map[string]string{ + v1alpha1.ControlledByPodOpsLifecycle: "true", fmt.Sprintf("%s/%s", v1alpha1.PodOperateLabelPrefix, id): time, fmt.Sprintf("%s/%s", v1alpha1.PodCompleteLabelPrefix, id): time, }, @@ -247,6 +253,7 @@ var _ = Describe("podopslifecycle controller", func() { ObjectMeta: metav1.ObjectMeta{ Name: "test", Namespace: "default", + Labels: map[string]string{v1alpha1.ControlledByPodOpsLifecycle: "true"}, }, Spec: podSpec, } @@ -261,9 +268,9 @@ var _ = Describe("podopslifecycle controller", func() { Namespace: "default", }, pod) Expect(err).NotTo(HaveOccurred()) - Expect(pod.Status.Conditions).To(HaveLen(0)) pod.ObjectMeta.Labels = map[string]string{ + v1alpha1.ControlledByPodOpsLifecycle: "true", fmt.Sprintf("%s/%s", v1alpha1.PodOperateLabelPrefix, id): time, fmt.Sprintf("%s/%s", v1alpha1.PodCompleteLabelPrefix, id): time, } diff --git a/pkg/controllers/podopslifecycle/predicate.go b/pkg/controllers/podopslifecycle/predicate.go index d1f3b7ec..194a43e2 100644 --- a/pkg/controllers/podopslifecycle/predicate.go +++ b/pkg/controllers/podopslifecycle/predicate.go @@ -18,12 +18,11 @@ package podopslifecycle import ( corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" "sigs.k8s.io/controller-runtime/pkg/event" ) -type NeedOpsLifecycle func(oldPod, newPod *v1.Pod) bool +type NeedOpsLifecycle func(oldPod, newPod *corev1.Pod) bool type PodPredicate struct { NeedOpsLifecycle // check if pod need use lifecycle @@ -31,7 +30,7 @@ type PodPredicate struct { func (pp *PodPredicate) Create(evt event.CreateEvent) bool { if pp.NeedOpsLifecycle == nil { - return true + return false } pod := evt.Object.(*corev1.Pod) @@ -44,7 +43,7 @@ func (pp *PodPredicate) Delete(evt event.DeleteEvent) bool { func (pp *PodPredicate) Update(evt event.UpdateEvent) bool { if pp.NeedOpsLifecycle == nil { - return true + return false } oldPod := evt.ObjectOld.(*corev1.Pod) @@ -56,18 +55,12 @@ func (pp *PodPredicate) Update(evt event.UpdateEvent) bool { return false } - rv := oldPod.ResourceVersion - defer func() { - oldPod.ResourceVersion = rv - }() - - oldPod.ResourceVersion = newPod.ResourceVersion - return !equality.Semantic.DeepEqual(oldPod.ObjectMeta, newPod.ObjectMeta) + return !equality.Semantic.DeepEqual(oldPod.ObjectMeta.Annotations, newPod.ObjectMeta.Annotations) || !equality.Semantic.DeepEqual(oldPod.ObjectMeta.Labels, newPod.ObjectMeta.Labels) } func (pp *PodPredicate) Generic(evt event.GenericEvent) bool { if pp.NeedOpsLifecycle == nil { - return true + return false } pod := evt.Object.(*corev1.Pod) diff --git a/pkg/controllers/ruleset/checker/checker.go b/pkg/controllers/ruleset/checker/checker.go index 58853ea9..61634733 100644 --- a/pkg/controllers/ruleset/checker/checker.go +++ b/pkg/controllers/ruleset/checker/checker.go @@ -89,6 +89,9 @@ type CheckState struct { } func (cs *CheckState) InStage(stage string) bool { + if len(cs.States) == 0 { + return false + } for _, state := range cs.States { if state.Detail.Stage != stage { return false diff --git a/pkg/controllers/utils/expectations/expectation.go b/pkg/controllers/utils/expectations/expectation.go index be5f7e6c..35450fc0 100644 --- a/pkg/controllers/utils/expectations/expectation.go +++ b/pkg/controllers/utils/expectations/expectation.go @@ -24,7 +24,6 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" - "kusionstack.io/kafed/pkg/log" ) const ( @@ -57,8 +56,7 @@ type ControllerExpectationsInterface interface { // ControllerExpectations is a cache mapping controllers to what they expect to see before being woken up for a sync. type ControllerExpectations struct { cache.Store - name string - logger *log.Logger + name string } // GetExpectations returns the ControlleeExpectations of the given controller. @@ -85,13 +83,13 @@ func (r *ControllerExpectations) DeleteExpectations(controllerKey string) { func (r *ControllerExpectations) SatisfiedExpectations(controllerKey string) bool { if exp, exists, err := r.GetExpectations(controllerKey); exists { if exp.Fulfilled() { - r.logger.V(4).Infof("Controller expectations fulfilled %s", controllerKey) + klog.Infof("Controller expectations fulfilled %s", controllerKey) return true } else if exp.isExpired() { - r.logger.Errorf("expectation expired for key %s", controllerKey) + klog.Errorf("expectation expired for key %s", controllerKey) panic(fmt.Sprintf("expected panic for expectation [%s] timeout for key %s", r.name, controllerKey)) } else { - r.logger.V(4).Infof("Controller still waiting on expectations %#v", controllerKey) + klog.Infof("Controller still waiting on expectations %#v", controllerKey) return false } } else if err != nil { @@ -204,8 +202,8 @@ func (e *ControlleeExpectations) GetExpectations() (int64, int64) { } // NewControllerExpectations returns a store for ControllerExpectations. -func NewControllerExpectations(name string, logger *log.Logger) *ControllerExpectations { - return &ControllerExpectations{cache.NewStore(ExpKeyFunc), name, logger} +func NewControllerExpectations(name string) *ControllerExpectations { + return &ControllerExpectations{cache.NewStore(ExpKeyFunc), name} } // Expectations are a way for controllers to tell the controller manager what they expect. eg: diff --git a/pkg/controllers/utils/expectations/resourceversion_expectation.go b/pkg/controllers/utils/expectations/resourceversion_expectation.go index 2eb71414..e416d426 100644 --- a/pkg/controllers/utils/expectations/resourceversion_expectation.go +++ b/pkg/controllers/utils/expectations/resourceversion_expectation.go @@ -24,16 +24,14 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" - "kusionstack.io/kafed/pkg/log" ) type ResourceVersionExpectation struct { cache.Store - logger *log.Logger } -func NewResourceVersionExpectation(logger *log.Logger) *ResourceVersionExpectation { - return &ResourceVersionExpectation{cache.NewStore(ExpKeyFunc), logger} +func NewResourceVersionExpectation() *ResourceVersionExpectation { + return &ResourceVersionExpectation{cache.NewStore(ExpKeyFunc)} } func (r *ResourceVersionExpectation) GetExpectations(controllerKey string) (*ResourceVersionExpectationItem, bool, error) { @@ -55,13 +53,13 @@ func (r *ResourceVersionExpectation) DeleteExpectations(controllerKey string) { func (r *ResourceVersionExpectation) SatisfiedExpectations(controllerKey string, resourceVersion string) bool { if exp, exists, err := r.GetExpectations(controllerKey); exists { if exp.Fulfilled(resourceVersion) { - r.logger.V(4).Infof("Accuracy expectations fulfilled %s", controllerKey) + klog.Infof("Accuracy expectations fulfilled %s", controllerKey) return true } else if exp.isExpired() { - r.logger.Errorf("Accuracy expectation expired for key %s", controllerKey) + klog.Errorf("Accuracy expectation expired for key %s", controllerKey) panic(fmt.Sprintf("expected panic for accuracy expectation timeout for key %s", controllerKey)) } else { - r.logger.V(4).Infof("Controller still waiting on accuracy expectations %s", controllerKey) + klog.V(4).Infof("Controller still waiting on accuracy expectations %s", controllerKey) return false } } else if err != nil { diff --git a/pkg/log/logger.go b/pkg/utils/log/logger.go similarity index 100% rename from pkg/log/logger.go rename to pkg/utils/log/logger.go diff --git a/pkg/utils/pki_helpers.go b/pkg/utils/pki_helpers.go new file mode 100644 index 00000000..f2551ecd --- /dev/null +++ b/pkg/utils/pki_helpers.go @@ -0,0 +1,87 @@ +/* +Copyright 2019 The Kubernetes Authors. +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "crypto" + cryptorand "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math" + "math/big" + "time" + + "github.com/pkg/errors" + + certutil "k8s.io/client-go/util/cert" +) + +const ( + certificateBlockType = "CERTIFICATE" + rsaKeySize = 2048 + duration365d = time.Hour * 24 * 365 +) + +// NewPrivateKey creates an RSA private key +func NewPrivateKey() (*rsa.PrivateKey, error) { + return rsa.GenerateKey(cryptorand.Reader, rsaKeySize) +} + +// EncodeCertPEM returns PEM-endcoded certificate data +func EncodeCertPEM(cert *x509.Certificate) []byte { + block := pem.Block{ + Type: certificateBlockType, + Bytes: cert.Raw, + } + return pem.EncodeToMemory(&block) +} + +// NewSignedCert creates a signed certificate using the given CA certificate and key +func NewSignedCert(cfg *certutil.Config, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) { + serial, err := cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64)) + if err != nil { + return nil, err + } + if len(cfg.CommonName) == 0 { + return nil, errors.New("must specify a CommonName") + } + if len(cfg.Usages) == 0 { + return nil, errors.New("must specify at least one ExtKeyUsage") + } + + certTmpl := x509.Certificate{ + Subject: pkix.Name{ + CommonName: cfg.CommonName, + Organization: cfg.Organization, + }, + DNSNames: cfg.AltNames.DNSNames, + IPAddresses: cfg.AltNames.IPs, + SerialNumber: serial, + NotBefore: caCert.NotBefore, + NotAfter: time.Now().Add(duration365d).UTC(), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: cfg.Usages, + } + certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey) + if err != nil { + return nil, err + } + return x509.ParseCertificate(certDERBytes) +} diff --git a/pkg/webhook/server/generic/generic_mutating_handler.go b/pkg/webhook/server/generic/generic_mutating_handler.go index 1d972479..870246eb 100644 --- a/pkg/webhook/server/generic/generic_mutating_handler.go +++ b/pkg/webhook/server/generic/generic_mutating_handler.go @@ -20,6 +20,7 @@ import ( "context" "fmt" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -41,6 +42,8 @@ func (h *MutatingHandler) Handle(ctx context.Context, req admission.Request) (re if req.SubResource != "" { key = fmt.Sprintf("%s/%s", req.Kind.Kind, req.SubResource) } + klog.V(5).Infof("MutatingHandler, Kind: %s, Namespace: %s, Name: %s", key, req.Namespace, req.Name) + if handler, exist := MutatingTypeHandlerMap[key]; exist { return handler.Handle(ctx, req) } diff --git a/pkg/webhook/server/generic/generic_validating_handler.go b/pkg/webhook/server/generic/generic_validating_handler.go index 0d67437f..aee0167f 100644 --- a/pkg/webhook/server/generic/generic_validating_handler.go +++ b/pkg/webhook/server/generic/generic_validating_handler.go @@ -20,6 +20,7 @@ import ( "context" "fmt" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -38,6 +39,8 @@ func (h *ValidatingHandler) Handle(ctx context.Context, req admission.Request) ( if req.SubResource != "" { key = fmt.Sprintf("%s/%s", req.Kind.Kind, req.SubResource) } + klog.V(5).Infof("ValidatingHandler, Kind: %s, Namespace: %s, Name: %s", key, req.Namespace, req.Name) + if handler, exist := ValidatingTypeHandlerMap[key]; exist { return handler.Handle(ctx, req) } diff --git a/pkg/webhook/server/generic/pod/opslifecycle/mutating.go b/pkg/webhook/server/generic/pod/opslifecycle/mutating.go index dc1d2250..8e3b0bf9 100644 --- a/pkg/webhook/server/generic/pod/opslifecycle/mutating.go +++ b/pkg/webhook/server/generic/pod/opslifecycle/mutating.go @@ -22,6 +22,7 @@ import ( admissionv1 "k8s.io/api/admission/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "kusionstack.io/kafed/apis/apps/v1alpha1" @@ -30,20 +31,24 @@ import ( ) func (lc *OpsLifecycle) Mutating(ctx context.Context, c client.Client, oldPod, newPod *corev1.Pod, operation admissionv1.Operation) error { - if !utils.ControlledByPodOpsLifecycle(newPod) || operation != admissionv1.Update { + if !utils.ControlledByPodOpsLifecycle(newPod) { return nil } - addReadinessGates(newPod, v1alpha1.ReadinessGatePodServiceReady) + // add readiness gate when pod is created + if operation == admissionv1.Create { + addReadinessGates(newPod, v1alpha1.ReadinessGatePodServiceReady) + } - newIdToLabelsMap, typeToNumsMap, err := podopslifecycle.PodIDAndTypesMap(newPod) + newIDToLabelsMap, typeToNumsMap, err := podopslifecycle.PodIDAndTypesMap(newPod) if err != nil { return err } + numOfIDs := len(newIDToLabelsMap) - var operatingCount, operateCount, completeCount int + var operatingCount, operateCount, operatedCount, completeCount int var undoTypeToNumsMap = map[string]int{} - for id, labels := range newIdToLabelsMap { + for id, labels := range newIDToLabelsMap { if undoOperationType, ok := labels[v1alpha1.PodUndoOperationTypeLabelPrefix]; ok { // operation is canceled if _, ok := undoTypeToNumsMap[undoOperationType]; !ok { undoTypeToNumsMap[undoOperationType] = 1 @@ -58,6 +63,7 @@ func (lc *OpsLifecycle) Mutating(ctx context.Context, c client.Client, oldPod, n v1alpha1.PodPreCheckedLabelPrefix, v1alpha1.PodPrepareLabelPrefix, v1alpha1.PodOperateLabelPrefix} { + delete(newPod.Labels, fmt.Sprintf("%s/%s", v, id)) } @@ -68,42 +74,42 @@ func (lc *OpsLifecycle) Mutating(ctx context.Context, c client.Client, oldPod, n if _, ok := labels[v1alpha1.PodOperatingLabelPrefix]; ok { // operating operatingCount++ - if _, ok := labels[v1alpha1.PodPreCheckedLabelPrefix]; ok { + if _, ok := labels[v1alpha1.PodPreCheckedLabelPrefix]; ok { // pre-checked if _, ok := labels[v1alpha1.PodPrepareLabelPrefix]; !ok { - lc.addLabelWithTime(newPod, fmt.Sprintf("%s/%s", v1alpha1.PodPrepareLabelPrefix, id)) - } - if _, ok := labels[v1alpha1.PodOperateLabelPrefix]; !ok { + delete(newPod.Labels, v1alpha1.PodServiceAvailableLabel) + + lc.addLabelWithTime(newPod, fmt.Sprintf("%s/%s", v1alpha1.PodPrepareLabelPrefix, id)) // prepare + } else if _, ok := labels[v1alpha1.PodOperateLabelPrefix]; !ok { if ready, _ := lc.readyToUpgrade(newPod); ready { - lc.addLabelWithTime(newPod, fmt.Sprintf("%s/%s", v1alpha1.PodOperateLabelPrefix, id)) + delete(newPod.Labels, fmt.Sprintf("%s/%s", v1alpha1.PodPrepareLabelPrefix, id)) + + lc.addLabelWithTime(newPod, fmt.Sprintf("%s/%s", v1alpha1.PodOperateLabelPrefix, id)) // operate, controllers can begin to operate } } } else { if _, ok := labels[v1alpha1.PodPreCheckLabelPrefix]; !ok { - lc.addLabelWithTime(newPod, fmt.Sprintf("%s/%s", v1alpha1.PodPreCheckLabelPrefix, id)) + lc.addLabelWithTime(newPod, fmt.Sprintf("%s/%s", v1alpha1.PodPreCheckLabelPrefix, id)) // pre-check } } } + if _, ok := labels[v1alpha1.PodPostCheckedLabelPrefix]; ok { // post-checked + if _, ok := labels[v1alpha1.PodCompleteLabelPrefix]; !ok { + lc.addLabelWithTime(newPod, fmt.Sprintf("%s/%s", v1alpha1.PodCompleteLabelPrefix, id)) // complete, wait fo podopslifecycle controller adds readiness gate + } + } + if _, ok := labels[v1alpha1.PodOperateLabelPrefix]; ok { operateCount++ } - - if _, ok := labels[v1alpha1.PodOperatedLabelPrefix]; ok { // operated - if _, ok := labels[v1alpha1.PodPostCheckedLabelPrefix]; ok { - if _, ok := labels[v1alpha1.PodCompleteLabelPrefix]; !ok { - lc.addLabelWithTime(newPod, fmt.Sprintf("%s/%s", v1alpha1.PodCompleteLabelPrefix, id)) - } - } else { - if _, ok := labels[v1alpha1.PodPostCheckLabelPrefix]; !ok { - lc.addLabelWithTime(newPod, fmt.Sprintf("%s/%s", v1alpha1.PodPostCheckLabelPrefix, id)) - } - } + if _, ok := labels[v1alpha1.PodOperatedLabelPrefix]; ok { + operatedCount++ } - if _, ok := labels[v1alpha1.PodCompleteLabelPrefix]; ok { // complete completeCount++ } } + klog.Infof("pod: %s/%s, numOfIDs: %d, operatingCount: %d, operateCount: %d, operatedCount: %d, completeCount: %d", newPod.Namespace, newPod.Name, numOfIDs, operatingCount, operateCount, operatedCount, completeCount) for t, num := range undoTypeToNumsMap { if num == typeToNumsMap[t] { // reset the permission with type t if all operating with type t are canceled @@ -111,46 +117,51 @@ func (lc *OpsLifecycle) Mutating(ctx context.Context, c client.Client, oldPod, n } } - if operatingCount != 0 { // wait for all operations to be done + if operatingCount != 0 { // when operation is done, controller will remove operating label and operation type label return nil } - if completeCount == len(newIdToLabelsMap) { // all operations are done - satisfied, _, err := lc.satisfyExpectedFinalizers(newPod) - if err != nil { + if completeCount == numOfIDs { // all operations are completed + satisfied, expectedFinalizer, err := lc.satisfyExpectedFinalizers(newPod) // whether all expected finalizers are satisfied + if err != nil || !satisfied { + klog.Infof("pod: %s/%s, expected finalizers: %v, err: %v", newPod.Namespace, newPod.Name, expectedFinalizer, err) return err } - if satisfied { // all operations are done and all expected finalizers are satisfied, then remove all unuseful labels, and add service available label - for id := range newIdToLabelsMap { - for _, v := range []string{v1alpha1.PodOperateLabelPrefix, - v1alpha1.PodOperatedLabelPrefix, - v1alpha1.PodDoneOperationTypeLabelPrefix, - v1alpha1.PodPostCheckLabelPrefix, - v1alpha1.PodPostCheckedLabelPrefix, - v1alpha1.PodCompleteLabelPrefix} { - delete(newPod.Labels, fmt.Sprintf("%s/%s", v, id)) - } + if !lc.isPodReady(newPod) { + return nil + } + + // all operations are done and all expected finalizers are satisfied, then remove all unuseful labels, and add service available label + for id := range newIDToLabelsMap { + for _, v := range []string{v1alpha1.PodOperateLabelPrefix, + v1alpha1.PodOperatedLabelPrefix, + v1alpha1.PodDoneOperationTypeLabelPrefix, + v1alpha1.PodPostCheckLabelPrefix, + v1alpha1.PodPostCheckedLabelPrefix, + v1alpha1.PodCompleteLabelPrefix} { + + delete(newPod.Labels, fmt.Sprintf("%s/%s", v, id)) } - lc.addLabelWithTime(newPod, v1alpha1.PodServiceAvailableLabel) } + lc.addLabelWithTime(newPod, v1alpha1.PodServiceAvailableLabel) + return nil } - if operateCount == len(newIdToLabelsMap) { // all operations are prepared + if operateCount == numOfIDs { // all operations are going to be done oldIdToLabelsMap, _, err := podopslifecycle.PodIDAndTypesMap(oldPod) if err != nil { return err } - for id, labels := range newIdToLabelsMap { - for _, v := range []string{v1alpha1.PodPreCheckLabelPrefix, - v1alpha1.PodPreCheckedLabelPrefix, - v1alpha1.PodPrepareLabelPrefix} { + for id, labels := range newIDToLabelsMap { + for _, v := range []string{v1alpha1.PodPreCheckLabelPrefix, v1alpha1.PodPreCheckedLabelPrefix} { delete(newPod.Labels, fmt.Sprintf("%s/%s", v, id)) } if _, ok := labels[v1alpha1.PodOperatedLabelPrefix]; !ok { - lc.addLabelWithTime(newPod, fmt.Sprintf("%s/%s", v1alpha1.PodOperatedLabelPrefix, id)) + lc.addLabelWithTime(newPod, fmt.Sprintf("%s/%s", v1alpha1.PodOperatedLabelPrefix, id)) // operated + operatedCount++ } t, ok := oldIdToLabelsMap[id][v1alpha1.PodOperationTypeLabelPrefix] @@ -158,7 +169,16 @@ func (lc *OpsLifecycle) Mutating(ctx context.Context, c client.Client, oldPod, n continue } delete(newPod.Labels, fmt.Sprintf("%s/%s", v1alpha1.PodOperationPermissionLabelPrefix, t)) - newPod.Labels[fmt.Sprintf("%s/%s", v1alpha1.PodDoneOperationTypeLabelPrefix, id)] = t + + newPod.Labels[fmt.Sprintf("%s/%s", v1alpha1.PodDoneOperationTypeLabelPrefix, id)] = t // done-operation-type + } + } + + if operatedCount == numOfIDs { // all operations are done + for id, labels := range newIDToLabelsMap { + if _, ok := labels[v1alpha1.PodPostCheckLabelPrefix]; !ok { + lc.addLabelWithTime(newPod, fmt.Sprintf("%s/%s", v1alpha1.PodPostCheckLabelPrefix, id)) // post-check + } } } diff --git a/pkg/webhook/server/generic/pod/opslifecycle/webhook.go b/pkg/webhook/server/generic/pod/opslifecycle/webhook.go index d150eb00..21b162a4 100644 --- a/pkg/webhook/server/generic/pod/opslifecycle/webhook.go +++ b/pkg/webhook/server/generic/pod/opslifecycle/webhook.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "kusionstack.io/kafed/apis/apps/v1alpha1" + controllerutils "kusionstack.io/kafed/pkg/controllers/utils" ) const ( @@ -52,10 +53,12 @@ var ( type ReadyToUpgrade func(pod *corev1.Pod) (bool, []string) type SatisfyExpectedFinalizers func(pod *corev1.Pod) (bool, []string, error) type TimeLabelValue func() string +type IsPodReady func(pod *corev1.Pod) bool type OpsLifecycle struct { readyToUpgrade ReadyToUpgrade // for testing satisfyExpectedFinalizers SatisfyExpectedFinalizers + isPodReady IsPodReady timeLabelValue TimeLabelValue } @@ -63,6 +66,7 @@ func New() *OpsLifecycle { return &OpsLifecycle{ readyToUpgrade: hasNoBlockingFinalizer, satisfyExpectedFinalizers: satisfyExpectedFinalizers, + isPodReady: controllerutils.IsPodReady, timeLabelValue: func() string { return strconv.FormatInt(time.Now().Unix(), 10) }, @@ -92,12 +96,12 @@ func addReadinessGates(pod *corev1.Pod, conditionType corev1.PodConditionType) { } func satisfyExpectedFinalizers(pod *corev1.Pod) (bool, []string, error) { - satisfy := true + satisfied := true var expectedFinalizer []string // expected finalizers that are not satisfied availableConditions, err := podAvailableConditions(pod) if err != nil { - return satisfy, expectedFinalizer, err + return satisfied, expectedFinalizer, err } if availableConditions != nil && len(availableConditions.ExpectedFinalizers) != 0 { @@ -108,30 +112,13 @@ func satisfyExpectedFinalizers(pod *corev1.Pod) (bool, []string, error) { for _, finalizer := range availableConditions.ExpectedFinalizers { if !existFinalizers.Has(finalizer) { - satisfy = false + satisfied = false expectedFinalizer = append(expectedFinalizer, finalizer) } } } - return satisfy, expectedFinalizer, nil -} - -func podAvailableConditions(pod *corev1.Pod) (*v1alpha1.PodAvailableConditions, error) { - if pod.Annotations == nil { - return nil, nil - } - - anno, ok := pod.Annotations[v1alpha1.PodAvailableConditionsAnnotation] - if !ok { - return nil, nil - } - - availableConditions := &v1alpha1.PodAvailableConditions{} - if err := json.Unmarshal([]byte(anno), availableConditions); err != nil { - return nil, err - } - return availableConditions, nil + return satisfied, expectedFinalizer, nil } func hasNoBlockingFinalizer(pod *corev1.Pod) (bool, []string) { @@ -170,3 +157,20 @@ func hasNoBlockingFinalizer(pod *corev1.Pod) (bool, []string) { return true, nil } + +func podAvailableConditions(pod *corev1.Pod) (*v1alpha1.PodAvailableConditions, error) { + if pod.Annotations == nil { + return nil, nil + } + + anno, ok := pod.Annotations[v1alpha1.PodAvailableConditionsAnnotation] + if !ok { + return nil, nil + } + + availableConditions := &v1alpha1.PodAvailableConditions{} + if err := json.Unmarshal([]byte(anno), availableConditions); err != nil { + return nil, err + } + return availableConditions, nil +} diff --git a/pkg/webhook/server/generic/pod/opslifecycle/webhook_test.go b/pkg/webhook/server/generic/pod/opslifecycle/webhook_test.go index cd4c753d..65429efd 100644 --- a/pkg/webhook/server/generic/pod/opslifecycle/webhook_test.go +++ b/pkg/webhook/server/generic/pod/opslifecycle/webhook_test.go @@ -88,8 +88,7 @@ func TestValidating(t *testing.T) { }, { labels: map[string]string{ - v1alpha1.PodOperatingLabelPrefix: "1402144848", - fmt.Sprintf("%s/%s", v1alpha1.PodOperationTypeLabelPrefix, "123"): "upgrade", + v1alpha1.PodOperationTypeLabelPrefix: "1402144848", }, keyWords: "invalid label", }, @@ -127,6 +126,7 @@ func TestMutating(t *testing.T) { satisfyExpectedFinalizers SatisfyExpectedFinalizers readyToUpgrade ReadyToUpgrade + isPodReady IsPodReady }{ { notes: "pre-check", @@ -275,7 +275,6 @@ func TestMutating(t *testing.T) { fmt.Sprintf("%s/%s", v1alpha1.PodOperationPermissionLabelPrefix, "upgrade"): "1402144848", fmt.Sprintf("%s/%s", v1alpha1.PodPrepareLabelPrefix, "123"): "1402144848", - fmt.Sprintf("%s/%s", v1alpha1.PodOperateLabelPrefix, "123"): "1402144848", }, }, @@ -289,18 +288,17 @@ func TestMutating(t *testing.T) { fmt.Sprintf("%s/%s", v1alpha1.PodPreCheckedLabelPrefix, "123"): "1402144848", fmt.Sprintf("%s/%s", v1alpha1.PodOperationPermissionLabelPrefix, "upgrade"): "1402144848", fmt.Sprintf("%s/%s", v1alpha1.PodOperateLabelPrefix, "123"): "1402144848", - fmt.Sprintf("%s/%s", v1alpha1.PodPrepareLabelPrefix, "123"): "1402144848", }, newPodLabels: map[string]string{ fmt.Sprintf("%s/%s", v1alpha1.PodPreCheckLabelPrefix, "123"): "1402144848", fmt.Sprintf("%s/%s", v1alpha1.PodPreCheckedLabelPrefix, "123"): "1402144848", fmt.Sprintf("%s/%s", v1alpha1.PodOperationPermissionLabelPrefix, "upgrade"): "1402144848", fmt.Sprintf("%s/%s", v1alpha1.PodOperateLabelPrefix, "123"): "1402144848", - fmt.Sprintf("%s/%s", v1alpha1.PodPrepareLabelPrefix, "123"): "1402144848", }, expectedLabels: map[string]string{ fmt.Sprintf("%s/%s", v1alpha1.PodOperateLabelPrefix, "123"): "1402144848", fmt.Sprintf("%s/%s", v1alpha1.PodOperatedLabelPrefix, "123"): "1402144848", + fmt.Sprintf("%s/%s", v1alpha1.PodPostCheckLabelPrefix, "123"): "1402144848", fmt.Sprintf("%s/%s", v1alpha1.PodDoneOperationTypeLabelPrefix, "123"): "upgrade", }, }, @@ -432,11 +430,7 @@ func TestMutating(t *testing.T) { }, } - opslifecycle := &OpsLifecycle{ - timeLabelValue: func() string { - return "1402144848" - }, - } + opslifecycle := &OpsLifecycle{} for _, v := range inputs { if v.oldPodLabels != nil { v.oldPodLabels[v1alpha1.ControlledByPodOpsLifecycle] = "true" @@ -460,14 +454,15 @@ func TestMutating(t *testing.T) { }, } - opslifecycle.readyToUpgrade = v.readyToUpgrade - if opslifecycle.readyToUpgrade == nil { - opslifecycle.readyToUpgrade = readyToUpgradeReturnTrue + opsLifecycleDefaultFunc(opslifecycle) + if v.readyToUpgrade != nil { + opslifecycle.readyToUpgrade = v.readyToUpgrade } - - opslifecycle.satisfyExpectedFinalizers = v.satisfyExpectedFinalizers - if opslifecycle.satisfyExpectedFinalizers == nil { - opslifecycle.satisfyExpectedFinalizers = satifyExpectedFinalizersReturnTrue + if v.satisfyExpectedFinalizers != nil { + opslifecycle.satisfyExpectedFinalizers = v.satisfyExpectedFinalizers + } + if v.isPodReady != nil { + opslifecycle.isPodReady = v.isPodReady } t.Logf("notes: %s", v.notes) @@ -488,6 +483,16 @@ func TestMutating(t *testing.T) { } } +func opsLifecycleDefaultFunc(opslifecycle *OpsLifecycle) { + opslifecycle.timeLabelValue = func() string { + return "1402144848" + } + + opslifecycle.readyToUpgrade = readyToUpgradeReturnTrue + opslifecycle.satisfyExpectedFinalizers = satifyExpectedFinalizersReturnTrue + opslifecycle.isPodReady = isPodReadyReturnTrue +} + func readyToUpgradeReturnTrue(pod *corev1.Pod) (bool, []string) { return true, nil } @@ -503,3 +508,11 @@ func satifyExpectedFinalizersReturnTrue(pod *corev1.Pod) (bool, []string, error) func satifyExpectedFinalizersReturnFalse(pod *corev1.Pod) (bool, []string, error) { return false, nil, nil } + +func isPodReadyReturnTrue(pod *corev1.Pod) bool { + return true +} + +func isPodReadyReturnFalse(pod *corev1.Pod) bool { + return true +} diff --git a/pkg/webhook/server/generic/pod/pod_validating_handler.go b/pkg/webhook/server/generic/pod/pod_validating_handler.go index 4456ea18..1e19028e 100644 --- a/pkg/webhook/server/generic/pod/pod_validating_handler.go +++ b/pkg/webhook/server/generic/pod/pod_validating_handler.go @@ -47,21 +47,21 @@ func (h *ValidatingHandler) Handle(ctx context.Context, req admission.Request) ( pod := &corev1.Pod{} if err := h.Decode(req, pod); err != nil { s, _ := json.Marshal(req) - return admission.Errored(http.StatusBadRequest, fmt.Errorf("fail to decode old object from request %s: %s", s, err)) + return admission.Errored(http.StatusBadRequest, fmt.Errorf("failed to decode old object from request %s: %s", s, err)) } var oldPod *corev1.Pod if req.Operation == admissionv1.Update || req.Operation == admissionv1.Delete { oldPod = &corev1.Pod{} if err := h.DecodeRaw(req.OldObject, oldPod); err != nil { - return admission.Errored(http.StatusBadRequest, fmt.Errorf("fail to unmarshal old object: %s", err)) + return admission.Errored(http.StatusBadRequest, fmt.Errorf("failed to unmarshal old object: %s", err)) } } for _, webhook := range webhooks { if err := webhook.Validating(ctx, h.Client, oldPod, pod, req.Operation); err != nil { klog.Errorf("failed to validate pod, %v", err) - return admission.Denied(fmt.Sprintf("fail to validate %s, %v", webhook.Name(), err)) + return admission.Denied(fmt.Sprintf("failed to validate %s, %v", webhook.Name(), err)) } } diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go index 0ccf61b1..ca453b43 100644 --- a/pkg/webhook/webhook.go +++ b/pkg/webhook/webhook.go @@ -17,9 +17,33 @@ limitations under the License. package webhook import ( + "context" + "crypto" + "crypto/rsa" + "crypto/x509" + "os" + "path/filepath" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/util/cert" + "k8s.io/client-go/util/keyutil" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" + "kusionstack.io/kafed/pkg/utils" "sigs.k8s.io/controller-runtime/pkg/manager" ) +const ( + mutatingWebhookConfigurationName = "kusionstack-controller-manager-mutating" + validatingWebhookConfigurationName = "kusionstack-controller-manager-validating" + webhookCertsSecretName = "kusionstack-webhook-certs" +) + // AddToManagerFuncs is a list of functions to add all Webhook Servers to the Manager var AddToManagerFuncs []func(manager.Manager) error @@ -35,3 +59,199 @@ func AddToManager(m manager.Manager) error { } return nil } + +func Initialize(ctx context.Context, config *rest.Config, dnsName, certDir string) error { + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return err + } + return ensureWebhookCABundleAndCert(ctx, clientset, dnsName, certDir) +} + +func ensureWebhookCABundleAndCert(ctx context.Context, clientset *kubernetes.Clientset, dnsName, certDir string) error { + secret, err := ensureWebhookSecret(ctx, clientset, dnsName) + if err != nil { + return err + } + klog.Infof("webhook secret ensured, secret: %s", secret.Name) + + mwhc, err := clientset.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(ctx, mutatingWebhookConfigurationName, metav1.GetOptions{}) + if err != nil { + return err + } + + for i := range mwhc.Webhooks { + if mwhc.Webhooks[i].ClientConfig.CABundle == nil { + mwhc.Webhooks[i].ClientConfig.CABundle = secret.Data["ca.crt"] + } + } + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + _, err := clientset.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(ctx, mwhc, metav1.UpdateOptions{}) + return err + }) + if err != nil { + return err + } + + vwhc, err := clientset.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(ctx, validatingWebhookConfigurationName, metav1.GetOptions{}) + if err != nil { + return err + } + + for i := range vwhc.Webhooks { + if vwhc.Webhooks[i].ClientConfig.CABundle == nil { + vwhc.Webhooks[i].ClientConfig.CABundle = secret.Data["ca.crt"] + } + } + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + _, err := clientset.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update(ctx, vwhc, metav1.UpdateOptions{}) + return err + }) + if err != nil { + return err + } + klog.Infof("webhook ca bundle ensured, mutatingwebhookconfiguration: %s, validatingwebhookconfiguration: %s", mutatingWebhookConfigurationName, validatingWebhookConfigurationName) + + var tlsKey, tlsCert []byte + tlsKey, ok := secret.Data["tls.key"] + if !ok { + return errors.New("tls.key not found in secret") + } + tlsCert, ok = secret.Data["tls.crt"] + if !ok { + return errors.New("tls.crt not found in secret") + } + + err = ensureWebhookCert(certDir, tlsKey, tlsCert) + if err != nil { + return err + } + klog.Infof("webhook cert ensured, cert dir: %s", certDir) + + return nil +} + +func ensureWebhookSecret(ctx context.Context, clientset *kubernetes.Clientset, dnsName string) (secret *corev1.Secret, err error) { + var ( + found = true + dirty = false + ) + secret, err = clientset.CoreV1().Secrets(getNamespace()).Get(ctx, webhookCertsSecretName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + found = false + } else { + return + } + } + if found { + if secret.Data == nil || len(secret.Data) != 4 || + secret.Data["ca.key"] == nil || secret.Data["ca.crt"] == nil || + secret.Data["tls.key"] == nil || secret.Data["tls.crt"] == nil { + dirty = true + } + if !dirty { + return + } + } + + caKey, caCert, err := generateSelfSignedCACert() + if err != nil { + return + } + caKeyPEM, err := keyutil.MarshalPrivateKeyToPEM(caKey) + if err != nil { + return + } + caCertPEM := utils.EncodeCertPEM(caCert) + + privateKey, signedCert, err := generateSelfSignedCert(caCert, caKey, dnsName) + if err != nil { + return + } + privateKeyPEM, err := keyutil.MarshalPrivateKeyToPEM(privateKey) + if err != nil { + return + } + signedCertPEM := utils.EncodeCertPEM(signedCert) + + data := map[string][]byte{ + "ca.key": caKeyPEM, "ca.crt": caCertPEM, + "tls.key": privateKeyPEM, "tls.crt": signedCertPEM, + } + secret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: webhookCertsSecretName, + Namespace: getNamespace(), + }, + Data: data, + } + + var updatedSecret *corev1.Secret + err = retry.RetryOnConflict(retry.DefaultRetry, func() (err error) { + if dirty { + updatedSecret, err = clientset.CoreV1().Secrets(getNamespace()).Update(ctx, secret, metav1.UpdateOptions{}) + } else { + updatedSecret, err = clientset.CoreV1().Secrets(getNamespace()).Create(ctx, secret, metav1.CreateOptions{}) + } + return err + }) + return updatedSecret, err +} + +func generateSelfSignedCACert() (caKey *rsa.PrivateKey, caCert *x509.Certificate, err error) { + caKey, err = utils.NewPrivateKey() + if err != nil { + return + } + + caCert, err = cert.NewSelfSignedCACert(cert.Config{CommonName: "self-signed-k8s-cert"}, caKey) + + return +} + +func generateSelfSignedCert(caCert *x509.Certificate, caKey crypto.Signer, dnsName string) (privateKey *rsa.PrivateKey, signedCert *x509.Certificate, err error) { + privateKey, err = utils.NewPrivateKey() + if err != nil { + return + } + + signedCert, err = utils.NewSignedCert( + &cert.Config{ + CommonName: dnsName, + AltNames: cert.AltNames{DNSNames: []string{dnsName}}, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }, + privateKey, caCert, caKey, + ) + + return +} + +func ensureWebhookCert(certDir string, tlsKey, tlsCert []byte) error { + if _, err := os.Stat(certDir); os.IsNotExist(err) { + err := os.Mkdir(certDir, 0644) + if err != nil { + return err + } + klog.Infof("cert dir is created: %s", certDir) + } + + keyFile := filepath.Join(certDir, "tls.key") + certFile := filepath.Join(certDir, "tls.crt") + + if err := os.WriteFile(keyFile, tlsKey, 0644); err != nil { + return err + } + if err := os.WriteFile(certFile, tlsCert, 0644); err != nil { + return err + } + return nil +} + +func getNamespace() string { + if ns := os.Getenv("POD_NAMESPACE"); len(ns) > 0 { + return ns + } + return "kusionstack-system" +} diff --git a/test/e2e/scripts/kind-conf.yaml b/test/e2e/scripts/kind-conf.yaml index e7c9521f..d8f00efe 100644 --- a/test/e2e/scripts/kind-conf.yaml +++ b/test/e2e/scripts/kind-conf.yaml @@ -1,6 +1,6 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 -name: kindcluster +name: cluster1 nodes: - role: control-plane - role: worker