From 6d3d8271caaff43f3347c2bc313fc1b6e0f65204 Mon Sep 17 00:00:00 2001 From: Senthil Raja Chermapandian Date: Wed, 30 Jun 2021 15:37:12 +0530 Subject: [PATCH 01/19] fix issue #75: workaround --- README.md | 2 +- .../helm-charts/kubefledged/templates/validatingwebhook.yaml | 2 +- deploy/webhook-create-signed-cert.sh | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 18604c96..7c263ce8 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,7 @@ These instructions install _kube-fledged_ to a separate namespace called "kube-f - Verify if _kube-fledged_ deployed successfully ``` - $ kubectl get pods -n kube-fledged -l app.kubernetes.io/name=kubefledged + $ kubectl get pods -n kube-fledged -l app.kubernetes.io/name=kube-fledged $ kubectl logs -f -n kube-fledged $ kubectl get imagecaches -n kube-fledged (Output should be: 'No resources found') ``` diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml index 56a3e1ba..e3403867 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml @@ -12,7 +12,7 @@ webhooks: clientConfig: service: namespace: {{ .Values.kubefledgedNameSpace }} - name: kubefledged-webhook-server + name: {{ include "kubefledged.fullname" . }}-webhook-server path: "/validate-image-cache" port: {{ .Values.webhookService.port }} caBundle: {{ .Values.validatingWebhookCABundle }} diff --git a/deploy/webhook-create-signed-cert.sh b/deploy/webhook-create-signed-cert.sh index 9a73ddc2..f870dfb2 100644 --- a/deploy/webhook-create-signed-cert.sh +++ b/deploy/webhook-create-signed-cert.sh @@ -101,7 +101,8 @@ openssl req -new -key ${tmpdir}/server-key.pem -subj "/CN=${service}.${namespace kubectl delete csr ${csrName} 2>/dev/null || true # create server cert/key CSR and send to k8s API -apiVersion=$(kubectl api-versions | grep certificates.k8s.io | head -1) +#apiVersion=$(kubectl api-versions | grep certificates.k8s.io | head -1) +apiVersion=certificates.k8s.io/v1beta1 cat < Date: Fri, 9 Jul 2021 18:29:18 -0600 Subject: [PATCH 02/19] Ensure validating webhook configuration client config service name for the webhook server mataches the correct webhook service name --- .../kubefledged-operator/helm-charts/kubefledged/Chart.yaml | 2 +- .../helm-charts/kubefledged/templates/_helpers.tpl | 2 +- .../kubefledged/templates/service-webhook-server.yaml | 2 +- .../helm-charts/kubefledged/templates/validatingwebhook.yaml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/Chart.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/Chart.yaml index 24ee29a5..4967f6ea 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/Chart.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/Chart.yaml @@ -14,7 +14,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: v0.8.0 +version: v0.8.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/_helpers.tpl b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/_helpers.tpl index ce5116f2..262af0f8 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/_helpers.tpl +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/_helpers.tpl @@ -100,7 +100,7 @@ Create the name of the service for the webhook server to use */}} {{- define "kubefledged.webhookServiceName" -}} {{- if .Values.webhookService.create -}} - {{ default (include "kubefledged.fullname" .) .Values.webhookService.name }} + {{ default ( printf "%s-webhook-server" (include "kubefledged.fullname" .)) .Values.webhookService.name }} {{- else -}} {{ default "default" .Values.webhookService.name }} {{- end -}} diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/service-webhook-server.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/service-webhook-server.yaml index 8532616d..65b4af2e 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/service-webhook-server.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/service-webhook-server.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "kubefledged.fullname" . }}-webhook-server + name: {{ include "kubefledged.webhookServiceName" . }} labels: {{ include "kubefledged.labels" . | nindent 4 }} namespace: {{ .Values.kubefledgedNameSpace }} diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml index e3403867..2b66e6d6 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml @@ -12,7 +12,7 @@ webhooks: clientConfig: service: namespace: {{ .Values.kubefledgedNameSpace }} - name: {{ include "kubefledged.fullname" . }}-webhook-server + name: {{ include "kubefledged.webhookServiceName" . }} path: "/validate-image-cache" port: {{ .Values.webhookService.port }} caBundle: {{ .Values.validatingWebhookCABundle }} @@ -22,4 +22,4 @@ webhooks: apiVersions: ["v1alpha2"] resources: ["imagecaches"] scope: "Namespaced" -{{- end -}} \ No newline at end of file +{{- end -}} From 24aadcda7d7964c88e1527a76ac3a0df3d5ecc80 Mon Sep 17 00:00:00 2001 From: Senthil Raja Chermapandian Date: Thu, 22 Jul 2021 14:05:51 +0530 Subject: [PATCH 03/19] add init option to webhook server --- cmd/webhook-server/app/init.go | 214 ++++++++++++++++++++++++ cmd/webhook-server/app/server.go | 274 +++++++++++++++++++++++++++++++ cmd/webhook-server/main.go | 267 ++---------------------------- 3 files changed, 506 insertions(+), 249 deletions(-) create mode 100644 cmd/webhook-server/app/init.go create mode 100644 cmd/webhook-server/app/server.go diff --git a/cmd/webhook-server/app/init.go b/cmd/webhook-server/app/init.go new file mode 100644 index 00000000..a7701c59 --- /dev/null +++ b/cmd/webhook-server/app/init.go @@ -0,0 +1,214 @@ +/* +Copyright 2018 The kube-fledged authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "bytes" + "context" + cryptorand "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/json" + "encoding/pem" + "math/big" + "os" + "time" + + "github.com/golang/glog" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +func InitWebhookServer() error { + var caPEM, serverCertPEM, serverPrivKeyPEM *bytes.Buffer + + webhookServerService := os.Getenv("WEBHOOK_SERVER_SERVICE") + webhookServerNameSpace := os.Getenv("WEBHOOK_SERVER_NAME_SPACE") + certKeyPath := "/etc/webhook/certs/" + validatingWebhookConfig := os.Getenv("VALIDATING_WEBHOOK_CONFIG") + + // CA config + caConf := &x509.Certificate{ + SerialNumber: big.NewInt(2021), + Subject: pkix.Name{ + Organization: []string{"kubefledged.io"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(1, 0, 0), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + // CA private key + caPrivKey, err := rsa.GenerateKey(cryptorand.Reader, 4096) + if err != nil { + glog.Errorf("error in generating CA private key: %v", err) + return err + } + + // Self signed CA certificate + caBytes, err := x509.CreateCertificate(cryptorand.Reader, caConf, caConf, &caPrivKey.PublicKey, caPrivKey) + if err != nil { + glog.Errorf("error in generating CA certificate: %v", err) + return err + } + + // PEM encode CA cert + caPEM = new(bytes.Buffer) + _ = pem.Encode(caPEM, &pem.Block{ + Type: "CERTIFICATE", + Bytes: caBytes, + }) + + dnsNames := []string{ + webhookServerService, + webhookServerService + "." + webhookServerNameSpace, + webhookServerService + "." + webhookServerNameSpace + ".svc", + webhookServerService + "." + webhookServerNameSpace + ".svc.cluster"} + commonName := webhookServerService + "." + webhookServerNameSpace + ".svc" + + // server cert config + certConf := &x509.Certificate{ + DNSNames: dnsNames, + SerialNumber: big.NewInt(1658), + Subject: pkix.Name{ + CommonName: commonName, + Organization: []string{"kubefledged.io"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(1, 0, 0), + SubjectKeyId: []byte{1, 2, 3, 4, 6}, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature, + } + + // server private key + serverPrivKey, err := rsa.GenerateKey(cryptorand.Reader, 4096) + if err != nil { + glog.Errorf("error in generating server private key: %v", err) + return err + } + + // sign the server cert + serverCertBytes, err := x509.CreateCertificate(cryptorand.Reader, certConf, caConf, &serverPrivKey.PublicKey, caPrivKey) + if err != nil { + glog.Errorf("error in generating server certificate: %v", err) + return err + } + + // PEM encode the server cert and key + serverCertPEM = new(bytes.Buffer) + _ = pem.Encode(serverCertPEM, &pem.Block{ + Type: "CERTIFICATE", + Bytes: serverCertBytes, + }) + + serverPrivKeyPEM = new(bytes.Buffer) + _ = pem.Encode(serverPrivKeyPEM, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(serverPrivKey), + }) + + err = os.MkdirAll(certKeyPath, 0666) + if err != nil { + glog.Errorf("error in creating directory %s: %v", certKeyPath, err) + return err + } + err = writeFile(certKeyPath+"tls.crt", serverCertPEM) + if err != nil { + glog.Errorf("error in writing tls.crt: %v", err) + return err + } + + err = writeFile(certKeyPath+"tls.key", serverPrivKeyPEM) + if err != nil { + glog.Errorf("error in writing tls.key: %v", err) + return err + } + + err = patchValidatingWebhookConfig(caPEM, validatingWebhookConfig) + if err != nil { + return err + } + return nil +} + +// writeFile writes data in the file at the given path +func writeFile(filepath string, sCert *bytes.Buffer) error { + f, err := os.Create(filepath) + if err != nil { + return err + } + defer f.Close() + + _, err = f.Write(sCert.Bytes()) + if err != nil { + return err + } + return nil +} + +func patchValidatingWebhookConfig(caPEM *bytes.Buffer, validatingWebhookConfig string) error { + + cfg, err := rest.InClusterConfig() + if err != nil { + glog.Fatalf("Error building kubeconfig: %s", err.Error()) + return err + } + + kubeClient, err := kubernetes.NewForConfig(cfg) + if err != nil { + glog.Fatalf("Error building kubernetes clientset: %s", err.Error()) + return err + } + + _, err = kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get( + context.TODO(), validatingWebhookConfig, metav1.GetOptions{}) + if err != nil { + glog.Errorf("Error in getting validatingwebhookconfig: %s", err.Error()) + return err + } + + // patchStringValue specifies a patch operation for a string. + type patchStringValue struct { + Op string `json:"op"` + Path string `json:"path"` + Value string `json:"value"` + } + payload := []patchStringValue{{ + Op: "replace", + Path: "/webhooks/0/clientConfig/caBundle", + Value: base64.StdEncoding.EncodeToString(caPEM.Bytes()), + }} + payloadBytes, _ := json.Marshal(payload) + + _, err = kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Patch( + context.TODO(), validatingWebhookConfig, types.JSONPatchType, payloadBytes, + metav1.PatchOptions{}) + if err != nil { + glog.Errorf("Error in patching validatingwebhookconfig: %s", err.Error()) + return err + } + + return nil +} diff --git a/cmd/webhook-server/app/server.go b/cmd/webhook-server/app/server.go new file mode 100644 index 00000000..58284fb5 --- /dev/null +++ b/cmd/webhook-server/app/server.go @@ -0,0 +1,274 @@ +/* +Copyright 2018 The kube-fledged authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + + "github.com/golang/glog" + "github.com/senthilrch/kube-fledged/pkg/webhook" + + admissionv1 "k8s.io/api/admission/v1" + admissionv1beta1 "k8s.io/api/admission/v1beta1" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + // TODO: try this library to see if it generates correct json patch + // https://github.com/mattbaird/jsonpatch +) + +var scheme = runtime.NewScheme() +var codecs = serializer.NewCodecFactory(scheme) + +func addToScheme(scheme *runtime.Scheme) { + utilruntime.Must(corev1.AddToScheme(scheme)) + utilruntime.Must(admissionv1beta1.AddToScheme(scheme)) + utilruntime.Must(admissionregistrationv1beta1.AddToScheme(scheme)) + utilruntime.Must(admissionv1.AddToScheme(scheme)) + utilruntime.Must(admissionregistrationv1.AddToScheme(scheme)) +} + +func init() { + addToScheme(scheme) +} + +// admitv1beta1Func handles a v1beta1 admission +type admitv1beta1Func func(admissionv1beta1.AdmissionReview) *admissionv1beta1.AdmissionResponse + +// admitv1beta1Func handles a v1 admission +type admitv1Func func(admissionv1.AdmissionReview) *admissionv1.AdmissionResponse + +// admitHandler is a handler, for both validators and mutators, that supports multiple admission review versions +type admitHandler struct { + v1beta1 admitv1beta1Func + v1 admitv1Func +} + +// Config contains the server (the webhook) cert and key. +type Config struct { + CertFile string + KeyFile string +} + +func configTLS(config Config) *tls.Config { + sCert, err := tls.LoadX509KeyPair(config.CertFile, config.KeyFile) + if err != nil { + glog.Fatal(err) + } + return &tls.Config{ + Certificates: []tls.Certificate{sCert}, + // TODO: uses mutual tls after we agree on what cert the apiserver should use. + // ClientAuth: tls.RequireAndVerifyClientCert, + } +} + +func newDelegateToV1AdmitHandler(f admitv1Func) admitHandler { + return admitHandler{ + v1beta1: delegateV1beta1AdmitToV1(f), + v1: f, + } +} + +func delegateV1beta1AdmitToV1(f admitv1Func) admitv1beta1Func { + return func(review admissionv1beta1.AdmissionReview) *admissionv1beta1.AdmissionResponse { + in := admissionv1.AdmissionReview{Request: convertAdmissionRequestToV1(review.Request)} + out := f(in) + return convertAdmissionResponseToV1beta1(out) + } +} + +// serve handles the http portion of a request prior to handing to an admit +// function +func serve(w http.ResponseWriter, r *http.Request, admit admitHandler) { + var body []byte + if r.Body != nil { + if data, err := ioutil.ReadAll(r.Body); err == nil { + body = data + } + } + + // verify the content type is accurate + contentType := r.Header.Get("Content-Type") + if contentType != "application/json" { + glog.Errorf("contentType=%s, expect application/json", contentType) + return + } + + glog.V(2).Info(fmt.Sprintf("handling request: %s", body)) + + deserializer := codecs.UniversalDeserializer() + obj, gvk, err := deserializer.Decode(body, nil, nil) + if err != nil { + msg := fmt.Sprintf("Request could not be decoded: %v", err) + glog.Error(msg) + http.Error(w, msg, http.StatusBadRequest) + return + } + + var responseObj runtime.Object + switch *gvk { + case admissionv1beta1.SchemeGroupVersion.WithKind("AdmissionReview"): + requestedAdmissionReview, ok := obj.(*admissionv1beta1.AdmissionReview) + if !ok { + glog.Errorf("Expected v1beta1.AdmissionReview but got: %T", obj) + return + } + responseAdmissionReview := &admissionv1beta1.AdmissionReview{} + responseAdmissionReview.SetGroupVersionKind(*gvk) + responseAdmissionReview.Response = admit.v1beta1(*requestedAdmissionReview) + responseAdmissionReview.Response.UID = requestedAdmissionReview.Request.UID + responseObj = responseAdmissionReview + case admissionv1.SchemeGroupVersion.WithKind("AdmissionReview"): + requestedAdmissionReview, ok := obj.(*admissionv1.AdmissionReview) + if !ok { + glog.Errorf("Expected v1.AdmissionReview but got: %T", obj) + return + } + responseAdmissionReview := &admissionv1.AdmissionReview{} + responseAdmissionReview.SetGroupVersionKind(*gvk) + responseAdmissionReview.Response = admit.v1(*requestedAdmissionReview) + responseAdmissionReview.Response.UID = requestedAdmissionReview.Request.UID + responseObj = responseAdmissionReview + default: + msg := fmt.Sprintf("Unsupported group version kind: %v", gvk) + glog.Error(msg) + http.Error(w, msg, http.StatusBadRequest) + return + } + + glog.V(2).Info(fmt.Sprintf("sending response: %v", responseObj)) + respBytes, err := json.Marshal(responseObj) + if err != nil { + glog.Error(err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + if _, err := w.Write(respBytes); err != nil { + glog.Error(err) + } +} + +func convertAdmissionRequestToV1(r *admissionv1beta1.AdmissionRequest) *admissionv1.AdmissionRequest { + return &admissionv1.AdmissionRequest{ + Kind: r.Kind, + Namespace: r.Namespace, + Name: r.Name, + Object: r.Object, + Resource: r.Resource, + Operation: admissionv1.Operation(r.Operation), + UID: r.UID, + DryRun: r.DryRun, + OldObject: r.OldObject, + Options: r.Options, + RequestKind: r.RequestKind, + RequestResource: r.RequestResource, + RequestSubResource: r.RequestSubResource, + SubResource: r.SubResource, + UserInfo: r.UserInfo, + } +} + +/* +func convertAdmissionRequestToV1beta1(r *admissionv1.AdmissionRequest) *admissionv1beta1.AdmissionRequest { + return &admissionv1beta1.AdmissionRequest{ + Kind: r.Kind, + Namespace: r.Namespace, + Name: r.Name, + Object: r.Object, + Resource: r.Resource, + Operation: admissionv1beta1.Operation(r.Operation), + UID: r.UID, + DryRun: r.DryRun, + OldObject: r.OldObject, + Options: r.Options, + RequestKind: r.RequestKind, + RequestResource: r.RequestResource, + RequestSubResource: r.RequestSubResource, + SubResource: r.SubResource, + UserInfo: r.UserInfo, + } +} + +func convertAdmissionResponseToV1(r *admissionv1beta1.AdmissionResponse) *admissionv1.AdmissionResponse { + var pt *admissionv1.PatchType + if r.PatchType != nil { + t := admissionv1.PatchType(*r.PatchType) + pt = &t + } + return &admissionv1.AdmissionResponse{ + UID: r.UID, + Allowed: r.Allowed, + AuditAnnotations: r.AuditAnnotations, + Patch: r.Patch, + PatchType: pt, + Result: r.Result, + } +} +*/ + +func convertAdmissionResponseToV1beta1(r *admissionv1.AdmissionResponse) *admissionv1beta1.AdmissionResponse { + var pt *admissionv1beta1.PatchType + if r.PatchType != nil { + t := admissionv1beta1.PatchType(*r.PatchType) + pt = &t + } + return &admissionv1beta1.AdmissionResponse{ + UID: r.UID, + Allowed: r.Allowed, + AuditAnnotations: r.AuditAnnotations, + Patch: r.Patch, + PatchType: pt, + Result: r.Result, + } +} + +func validateImageCache(w http.ResponseWriter, r *http.Request) { + serve(w, r, newDelegateToV1AdmitHandler(webhook.ValidateImageCache)) +} + +func mutateImageCache(w http.ResponseWriter, r *http.Request) { + // serve(w, r, newDelegateToV1AdmitHandler(webhook.MutateImageCache)) +} + +func StartWebhookServer(certFile string, keyFile string, port int) error { + config := Config{ + CertFile: certFile, + KeyFile: keyFile, + } + + http.HandleFunc("/validate-image-cache", validateImageCache) + http.HandleFunc("/mutate-image-cache", mutateImageCache) + http.HandleFunc("/readyz", func(w http.ResponseWriter, req *http.Request) { w.Write([]byte("ok")) }) + server := &http.Server{ + Addr: fmt.Sprintf(":%d", port), + TLSConfig: configTLS(config), + } + glog.Infof("Wehook server listening on :%d", port) + err := server.ListenAndServeTLS("", "") + if err != nil { + return err + } + return nil +} diff --git a/cmd/webhook-server/main.go b/cmd/webhook-server/main.go index b28f47ab..644f21f6 100644 --- a/cmd/webhook-server/main.go +++ b/cmd/webhook-server/main.go @@ -17,271 +17,40 @@ limitations under the License. package main import ( - "crypto/tls" - "encoding/json" "flag" - "fmt" - "io/ioutil" - "net/http" - "github.com/golang/glog" - "github.com/senthilrch/kube-fledged/pkg/webhook" - - admissionv1 "k8s.io/api/admission/v1" - v1 "k8s.io/api/admission/v1" - "k8s.io/api/admission/v1beta1" - admissionv1beta1 "k8s.io/api/admission/v1beta1" - admissionregistrationv1 "k8s.io/api/admissionregistration/v1" - admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - // TODO: try this library to see if it generates correct json patch - // https://github.com/mattbaird/jsonpatch + "github.com/senthilrch/kube-fledged/cmd/webhook-server/app" ) -var scheme = runtime.NewScheme() -var codecs = serializer.NewCodecFactory(scheme) - -func addToScheme(scheme *runtime.Scheme) { - utilruntime.Must(corev1.AddToScheme(scheme)) - utilruntime.Must(admissionv1beta1.AddToScheme(scheme)) - utilruntime.Must(admissionregistrationv1beta1.AddToScheme(scheme)) - utilruntime.Must(admissionv1.AddToScheme(scheme)) - utilruntime.Must(admissionregistrationv1.AddToScheme(scheme)) -} - -func init() { - addToScheme(scheme) -} - var ( - certFile string - keyFile string - port int + certFile string + keyFile string + port int + initServer bool ) func init() { flag.StringVar(&certFile, "cert-file", "", "File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated after server cert).") flag.StringVar(&keyFile, "key-file", "", "File containing the default x509 private key matching --cert-file.") flag.IntVar(&port, "port", 443, "Secure port that the webhook server listens on") -} - -// admitv1beta1Func handles a v1beta1 admission -type admitv1beta1Func func(v1beta1.AdmissionReview) *v1beta1.AdmissionResponse - -// admitv1beta1Func handles a v1 admission -type admitv1Func func(v1.AdmissionReview) *v1.AdmissionResponse - -// admitHandler is a handler, for both validators and mutators, that supports multiple admission review versions -type admitHandler struct { - v1beta1 admitv1beta1Func - v1 admitv1Func -} - -// Config contains the server (the webhook) cert and key. -type Config struct { - CertFile string - KeyFile string -} - -func configTLS(config Config) *tls.Config { - sCert, err := tls.LoadX509KeyPair(config.CertFile, config.KeyFile) - if err != nil { - glog.Fatal(err) - } - return &tls.Config{ - Certificates: []tls.Certificate{sCert}, - // TODO: uses mutual tls after we agree on what cert the apiserver should use. - // ClientAuth: tls.RequireAndVerifyClientCert, - } -} - -func newDelegateToV1AdmitHandler(f admitv1Func) admitHandler { - return admitHandler{ - v1beta1: delegateV1beta1AdmitToV1(f), - v1: f, - } -} - -func delegateV1beta1AdmitToV1(f admitv1Func) admitv1beta1Func { - return func(review v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { - in := v1.AdmissionReview{Request: convertAdmissionRequestToV1(review.Request)} - out := f(in) - return convertAdmissionResponseToV1beta1(out) - } -} - -// serve handles the http portion of a request prior to handing to an admit -// function -func serve(w http.ResponseWriter, r *http.Request, admit admitHandler) { - var body []byte - if r.Body != nil { - if data, err := ioutil.ReadAll(r.Body); err == nil { - body = data - } - } - - // verify the content type is accurate - contentType := r.Header.Get("Content-Type") - if contentType != "application/json" { - glog.Errorf("contentType=%s, expect application/json", contentType) - return - } - - glog.V(2).Info(fmt.Sprintf("handling request: %s", body)) - - deserializer := codecs.UniversalDeserializer() - obj, gvk, err := deserializer.Decode(body, nil, nil) - if err != nil { - msg := fmt.Sprintf("Request could not be decoded: %v", err) - glog.Error(msg) - http.Error(w, msg, http.StatusBadRequest) - return - } - - var responseObj runtime.Object - switch *gvk { - case v1beta1.SchemeGroupVersion.WithKind("AdmissionReview"): - requestedAdmissionReview, ok := obj.(*v1beta1.AdmissionReview) - if !ok { - glog.Errorf("Expected v1beta1.AdmissionReview but got: %T", obj) - return - } - responseAdmissionReview := &v1beta1.AdmissionReview{} - responseAdmissionReview.SetGroupVersionKind(*gvk) - responseAdmissionReview.Response = admit.v1beta1(*requestedAdmissionReview) - responseAdmissionReview.Response.UID = requestedAdmissionReview.Request.UID - responseObj = responseAdmissionReview - case v1.SchemeGroupVersion.WithKind("AdmissionReview"): - requestedAdmissionReview, ok := obj.(*v1.AdmissionReview) - if !ok { - glog.Errorf("Expected v1.AdmissionReview but got: %T", obj) - return - } - responseAdmissionReview := &v1.AdmissionReview{} - responseAdmissionReview.SetGroupVersionKind(*gvk) - responseAdmissionReview.Response = admit.v1(*requestedAdmissionReview) - responseAdmissionReview.Response.UID = requestedAdmissionReview.Request.UID - responseObj = responseAdmissionReview - default: - msg := fmt.Sprintf("Unsupported group version kind: %v", gvk) - glog.Error(msg) - http.Error(w, msg, http.StatusBadRequest) - return - } - - glog.V(2).Info(fmt.Sprintf("sending response: %v", responseObj)) - respBytes, err := json.Marshal(responseObj) - if err != nil { - glog.Error(err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "application/json") - if _, err := w.Write(respBytes); err != nil { - glog.Error(err) - } -} - -func convertAdmissionRequestToV1(r *v1beta1.AdmissionRequest) *v1.AdmissionRequest { - return &v1.AdmissionRequest{ - Kind: r.Kind, - Namespace: r.Namespace, - Name: r.Name, - Object: r.Object, - Resource: r.Resource, - Operation: v1.Operation(r.Operation), - UID: r.UID, - DryRun: r.DryRun, - OldObject: r.OldObject, - Options: r.Options, - RequestKind: r.RequestKind, - RequestResource: r.RequestResource, - RequestSubResource: r.RequestSubResource, - SubResource: r.SubResource, - UserInfo: r.UserInfo, - } -} - -func convertAdmissionRequestToV1beta1(r *v1.AdmissionRequest) *v1beta1.AdmissionRequest { - return &v1beta1.AdmissionRequest{ - Kind: r.Kind, - Namespace: r.Namespace, - Name: r.Name, - Object: r.Object, - Resource: r.Resource, - Operation: v1beta1.Operation(r.Operation), - UID: r.UID, - DryRun: r.DryRun, - OldObject: r.OldObject, - Options: r.Options, - RequestKind: r.RequestKind, - RequestResource: r.RequestResource, - RequestSubResource: r.RequestSubResource, - SubResource: r.SubResource, - UserInfo: r.UserInfo, - } -} - -func convertAdmissionResponseToV1(r *v1beta1.AdmissionResponse) *v1.AdmissionResponse { - var pt *v1.PatchType - if r.PatchType != nil { - t := v1.PatchType(*r.PatchType) - pt = &t - } - return &v1.AdmissionResponse{ - UID: r.UID, - Allowed: r.Allowed, - AuditAnnotations: r.AuditAnnotations, - Patch: r.Patch, - PatchType: pt, - Result: r.Result, - } -} - -func convertAdmissionResponseToV1beta1(r *v1.AdmissionResponse) *v1beta1.AdmissionResponse { - var pt *v1beta1.PatchType - if r.PatchType != nil { - t := v1beta1.PatchType(*r.PatchType) - pt = &t - } - return &v1beta1.AdmissionResponse{ - UID: r.UID, - Allowed: r.Allowed, - AuditAnnotations: r.AuditAnnotations, - Patch: r.Patch, - PatchType: pt, - Result: r.Result, - } -} - -func validateImageCache(w http.ResponseWriter, r *http.Request) { - serve(w, r, newDelegateToV1AdmitHandler(webhook.ValidateImageCache)) -} - -func mutateImageCache(w http.ResponseWriter, r *http.Request) { - // serve(w, r, newDelegateToV1AdmitHandler(webhook.MutateImageCache)) + flag.BoolVar(&initServer, "init-server", false, "True means only init tasks for the server will be performed. Server is not started") } func main() { flag.Parse() - config := Config{ - CertFile: certFile, - KeyFile: keyFile, - } - - http.HandleFunc("/validate-image-cache", validateImageCache) - http.HandleFunc("/mutate-image-cache", mutateImageCache) - http.HandleFunc("/readyz", func(w http.ResponseWriter, req *http.Request) { w.Write([]byte("ok")) }) - server := &http.Server{ - Addr: fmt.Sprintf(":%d", port), - TLSConfig: configTLS(config), + if initServer { + /* + Call function to perform init tasks: + - create CA cert and key + - create server cert and key and copy to /etc/webhook/certs + - patch validatingwebhookconfiguration with CA bundle + */ + if err := app.InitWebhookServer(); err != nil { + panic(err) + } + return } - glog.Infof("Wehook server listening on :%d", port) - err := server.ListenAndServeTLS("", "") - if err != nil { + if err := app.StartWebhookServer(certFile, keyFile, port); err != nil { panic(err) } } From 6bed39917808ef40ce5cdfe6ca3700312e178fe7 Mon Sep 17 00:00:00 2001 From: Senthil Raja Chermapandian Date: Thu, 22 Jul 2021 15:47:14 +0530 Subject: [PATCH 04/19] update manifests --- Makefile | 10 ++---- cmd/webhook-server/app/init.go | 4 +-- deploy/kubefledged-clusterrole.yaml | 20 +++++++++++- deploy/kubefledged-clusterrolebinding.yaml | 23 +++++++++++-- deploy/kubefledged-deployment-controller.yaml | 4 +-- ...kubefledged-deployment-webhook-server.yaml | 32 ++++++++++++++++--- .../kubefledged-operator/deploy/operator.yaml | 2 +- .../helm-charts/kubefledged/Chart.yaml | 2 +- deploy/kubefledged-serviceaccount.yaml | 9 ++++++ 9 files changed, 85 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 8b2d5f4d..89d65ed2 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ ifndef OPERATOR_IMAGE_REPO endif ifndef RELEASE_VERSION - RELEASE_VERSION=v0.8.0 + RELEASE_VERSION=v0.8.1 endif ifndef DOCKER_VERSION @@ -193,17 +193,15 @@ hack: deploy-using-yaml: -kubectl apply -f deploy/kubefledged-namespace.yaml - bash deploy/webhook-create-signed-cert.sh - bash deploy/webhook-patch-ca-bundle.sh kubectl apply -f deploy/kubefledged-crd.yaml kubectl apply -f deploy/kubefledged-serviceaccount.yaml kubectl apply -f deploy/kubefledged-clusterrole.yaml kubectl apply -f deploy/kubefledged-clusterrolebinding.yaml + kubectl apply -f deploy/kubefledged-validatingwebhook.yaml kubectl apply -f deploy/kubefledged-deployment-webhook-server.yaml kubectl rollout status deployment kubefledged-webhook-server -n kube-fledged --watch - kubectl apply -f deploy/kubefledged-deployment-controller.yaml kubectl apply -f deploy/kubefledged-service-webhook-server.yaml - kubectl apply -f deploy/kubefledged-validatingwebhook.yaml + kubectl apply -f deploy/kubefledged-deployment-controller.yaml deploy-using-operator: # Create the namespaces for operator and kubefledged @@ -221,8 +219,6 @@ deploy-using-operator: # Deploy kube-fledged to a separate namespace sed -i "s|{{OPERATOR_NAMESPACE}}|${OPERATOR_NAMESPACE}|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha2_kubefledged_cr.yaml sed -i "s|{{KUBEFLEDGED_NAMESPACE}}|${KUBEFLEDGED_NAMESPACE}|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha2_kubefledged_cr.yaml - bash deploy/webhook-create-signed-cert.sh --namespace ${KUBEFLEDGED_NAMESPACE} - bash deploy/webhook-patch-ca-bundle.sh kubectl apply -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha2_kubefledged_cr.yaml update: diff --git a/cmd/webhook-server/app/init.go b/cmd/webhook-server/app/init.go index a7701c59..951e83dd 100644 --- a/cmd/webhook-server/app/init.go +++ b/cmd/webhook-server/app/init.go @@ -41,8 +41,8 @@ func InitWebhookServer() error { var caPEM, serverCertPEM, serverPrivKeyPEM *bytes.Buffer webhookServerService := os.Getenv("WEBHOOK_SERVER_SERVICE") - webhookServerNameSpace := os.Getenv("WEBHOOK_SERVER_NAME_SPACE") - certKeyPath := "/etc/webhook/certs/" + webhookServerNameSpace := os.Getenv("KUBEFLEDGED_NAMESPACE") + certKeyPath := os.Getenv("CERT_KEY_PATH") validatingWebhookConfig := os.Getenv("VALIDATING_WEBHOOK_CONFIG") // CA config diff --git a/deploy/kubefledged-clusterrole.yaml b/deploy/kubefledged-clusterrole.yaml index ec713ad9..2b16bcc5 100644 --- a/deploy/kubefledged-clusterrole.yaml +++ b/deploy/kubefledged-clusterrole.yaml @@ -1,7 +1,7 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: kubefledged + name: kubefledged-controller labels: app: kubefledged component: kubefledged-controller @@ -58,3 +58,21 @@ rules: - list - watch - get +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kubefledged-webhook-server + labels: + app: kubefledged + component: kubefledged-webhook-server + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" +rules: + - apiGroups: + - "admissionregistration.k8s.io" + resources: + - validatingwebhookconfigurations + verbs: + - get + - patch diff --git a/deploy/kubefledged-clusterrolebinding.yaml b/deploy/kubefledged-clusterrolebinding.yaml index c809ce21..a7cbfb2f 100644 --- a/deploy/kubefledged-clusterrolebinding.yaml +++ b/deploy/kubefledged-clusterrolebinding.yaml @@ -1,14 +1,14 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: kubefledged + name: kubefledged-controller labels: app: kubefledged component: kubefledged-controller roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: kubefledged + name: kubefledged-controller subjects: - kind: ServiceAccount name: kubefledged-controller @@ -16,3 +16,22 @@ subjects: - apiGroup: rbac.authorization.k8s.io kind: Group name: system:nodes +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kubefledged-webhook-server + labels: + app: kubefledged + component: kubefledged-webhook-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubefledged-webhook-server +subjects: +- kind: ServiceAccount + name: kubefledged-webhook-server + namespace: kube-fledged +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes \ No newline at end of file diff --git a/deploy/kubefledged-deployment-controller.yaml b/deploy/kubefledged-deployment-controller.yaml index c3736d2f..a3afee6e 100644 --- a/deploy/kubefledged-deployment-controller.yaml +++ b/deploy/kubefledged-deployment-controller.yaml @@ -18,13 +18,13 @@ spec: app: kubefledged spec: containers: - - image: senthilrch/kubefledged-controller:v0.8.0 + - image: senthilrch/kubefledged-controller:v0.8.1 command: ["/opt/bin/kubefledged-controller"] args: - "--stderrthreshold=INFO" - "--image-pull-deadline-duration=5m" - "--image-cache-refresh-frequency=15m" - - "--cri-client-image=senthilrch/kubefledged-cri-client:v0.8.0" + - "--cri-client-image=senthilrch/kubefledged-cri-client:v0.8.1" - "--image-pull-policy=IfNotPresent" imagePullPolicy: Always name: controller diff --git a/deploy/kubefledged-deployment-webhook-server.yaml b/deploy/kubefledged-deployment-webhook-server.yaml index b5269031..d585862b 100644 --- a/deploy/kubefledged-deployment-webhook-server.yaml +++ b/deploy/kubefledged-deployment-webhook-server.yaml @@ -17,8 +17,31 @@ spec: kubefledged: kubefledged-webhook-server app: kubefledged spec: + initContainers: + - image: senthilrch/kubefledged-webhook-server:v0.8.1 + command: ["/opt/bin/kubefledged-webhook-server"] + args: + - "--stderrthreshold=INFO" + - "--init-server" + imagePullPolicy: Always + name: init + env: + - name: KUBEFLEDGED_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: WEBHOOK_SERVER_SERVICE + value: kubefledged-webhook-server + - name: VALIDATING_WEBHOOK_CONFIG + value: kubefledged + - name: CERT_KEY_PATH + value: "/var/run/secrets/webhook-server/" + volumeMounts: + - name: cert-key-volume + mountPath: "/var/run/secrets/webhook-server" + readOnly: true containers: - - image: senthilrch/kubefledged-webhook-server:v0.8.0 + - image: senthilrch/kubefledged-webhook-server:v0.8.1 command: ["/opt/bin/kubefledged-webhook-server"] args: - "--stderrthreshold=INFO" @@ -33,10 +56,9 @@ spec: fieldRef: fieldPath: metadata.namespace volumeMounts: - - name: secret-volume + - name: cert-key-volume mountPath: "/var/run/secrets/webhook-server" readOnly: true volumes: - - name: secret-volume - secret: - secretName: kubefledged-webhook-server + - name: cert-key-volume + emptyDir: {} diff --git a/deploy/kubefledged-operator/deploy/operator.yaml b/deploy/kubefledged-operator/deploy/operator.yaml index 5c7fb7b2..605485c5 100644 --- a/deploy/kubefledged-operator/deploy/operator.yaml +++ b/deploy/kubefledged-operator/deploy/operator.yaml @@ -17,7 +17,7 @@ spec: containers: - name: kubefledged-operator # Replace this with the built image name - image: docker.io/senthilrch/kubefledged-operator:v0.8.0 + image: docker.io/senthilrch/kubefledged-operator:v0.8.1 imagePullPolicy: Always env: - name: WATCH_NAMESPACE diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/Chart.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/Chart.yaml index 4967f6ea..f8df8752 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/Chart.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/Chart.yaml @@ -18,4 +18,4 @@ version: v0.8.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: v0.8.0 +appVersion: v0.8.1 diff --git a/deploy/kubefledged-serviceaccount.yaml b/deploy/kubefledged-serviceaccount.yaml index 1b201a64..bf9a8fb9 100644 --- a/deploy/kubefledged-serviceaccount.yaml +++ b/deploy/kubefledged-serviceaccount.yaml @@ -6,3 +6,12 @@ metadata: labels: app: kubefledged component: kubefledged-controller +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubefledged-webhook-server + namespace: kube-fledged + labels: + app: kubefledged + component: kubefledged-webhook-server \ No newline at end of file From 53d97b8f6bde1e69ff25fb756077d1b0cbc6c634 Mon Sep 17 00:00:00 2001 From: Senthil Raja Chermapandian Date: Thu, 22 Jul 2021 20:21:05 +0530 Subject: [PATCH 05/19] updated helm chart --- Makefile | 6 ++-- cmd/webhook-server/app/init.go | 29 +++++++++++++---- ...kubefledged-deployment-webhook-server.yaml | 12 +++---- .../charts.helm.k8s.io_kubefledgeds_crd.yaml | 14 ++++----- ...s.helm.k8s.io_v1alpha2_kubefledged_cr.yaml | 5 ++- ...rrole.yaml => clusterrole-controller.yaml} | 0 .../templates/clusterrole-webhook-server.yaml | 18 +++++++++++ ...aml => clusterrolebinding-controller.yaml} | 0 .../clusterrolebinding-webhook-server.yaml | 19 ++++++++++++ .../templates/deployment-controller.yaml | 2 +- .../templates/deployment-webhook-server.yaml | 31 ++++++++++++++++--- ...nt.yaml => serviceaccount-controller.yaml} | 0 .../serviceaccount-webhook-server.yaml | 9 ++++++ .../templates/validatingwebhook.yaml | 2 +- .../helm-charts/kubefledged/values.yaml | 4 +-- deploy/kubefledged-validatingwebhook.yaml | 2 +- 16 files changed, 117 insertions(+), 36 deletions(-) rename deploy/kubefledged-operator/helm-charts/kubefledged/templates/{clusterrole.yaml => clusterrole-controller.yaml} (100%) create mode 100644 deploy/kubefledged-operator/helm-charts/kubefledged/templates/clusterrole-webhook-server.yaml rename deploy/kubefledged-operator/helm-charts/kubefledged/templates/{clusterrolebinding.yaml => clusterrolebinding-controller.yaml} (100%) create mode 100644 deploy/kubefledged-operator/helm-charts/kubefledged/templates/clusterrolebinding-webhook-server.yaml rename deploy/kubefledged-operator/helm-charts/kubefledged/templates/{serviceaccount.yaml => serviceaccount-controller.yaml} (100%) create mode 100644 deploy/kubefledged-operator/helm-charts/kubefledged/templates/serviceaccount-webhook-server.yaml diff --git a/Makefile b/Makefile index 89d65ed2..73b323ae 100644 --- a/Makefile +++ b/Makefile @@ -202,6 +202,7 @@ deploy-using-yaml: kubectl rollout status deployment kubefledged-webhook-server -n kube-fledged --watch kubectl apply -f deploy/kubefledged-service-webhook-server.yaml kubectl apply -f deploy/kubefledged-deployment-controller.yaml + kubectl rollout status deployment kubefledged-controller -n kube-fledged --watch deploy-using-operator: # Create the namespaces for operator and kubefledged @@ -232,17 +233,14 @@ remove-kubefledged: -kubectl delete -f deploy/kubefledged-namespace.yaml -kubectl delete -f deploy/kubefledged-clusterrolebinding.yaml -kubectl delete -f deploy/kubefledged-clusterrole.yaml + -kubectl delete -f deploy/kubefledged-serviceaccount.yaml -kubectl delete -f deploy/kubefledged-crd.yaml -kubectl delete -f deploy/kubefledged-validatingwebhook.yaml - -git checkout deploy/kubefledged-validatingwebhook.yaml - -git checkout deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha2_kubefledged_cr.yaml remove-operator-and-kubefledged: # Remove kubefledged and the namespace -kubectl delete -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha2_kubefledged_cr.yaml -kubectl delete namespace ${KUBEFLEDGED_NAMESPACE} - -git checkout deploy/kubefledged-validatingwebhook.yaml - -git checkout deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha2_kubefledged_cr.yaml # Remove the kubefledged operator and the namespace -kubectl delete -f deploy/kubefledged-operator/deploy/operator.yaml -kubectl delete -f deploy/kubefledged-operator/deploy/clusterrole_binding.yaml diff --git a/cmd/webhook-server/app/init.go b/cmd/webhook-server/app/init.go index 951e83dd..ae88adaa 100644 --- a/cmd/webhook-server/app/init.go +++ b/cmd/webhook-server/app/init.go @@ -23,7 +23,6 @@ import ( "crypto/rsa" "crypto/x509" "crypto/x509/pkix" - "encoding/base64" "encoding/json" "encoding/pem" "math/big" @@ -54,7 +53,7 @@ func InitWebhookServer() error { NotBefore: time.Now(), NotAfter: time.Now().AddDate(1, 0, 0), IsCA: true, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, BasicConstraintsValid: true, } @@ -65,6 +64,7 @@ func InitWebhookServer() error { glog.Errorf("error in generating CA private key: %v", err) return err } + glog.Info("success: ca private key created") // Self signed CA certificate caBytes, err := x509.CreateCertificate(cryptorand.Reader, caConf, caConf, &caPrivKey.PublicKey, caPrivKey) @@ -72,6 +72,7 @@ func InitWebhookServer() error { glog.Errorf("error in generating CA certificate: %v", err) return err } + glog.Info("success: self-signed ca certificate created") // PEM encode CA cert caPEM = new(bytes.Buffer) @@ -79,6 +80,7 @@ func InitWebhookServer() error { Type: "CERTIFICATE", Bytes: caBytes, }) + glog.Info("success: ca certificate encoded to pem format") dnsNames := []string{ webhookServerService, @@ -98,7 +100,7 @@ func InitWebhookServer() error { NotBefore: time.Now(), NotAfter: time.Now().AddDate(1, 0, 0), SubjectKeyId: []byte{1, 2, 3, 4, 6}, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, KeyUsage: x509.KeyUsageDigitalSignature, } @@ -108,6 +110,7 @@ func InitWebhookServer() error { glog.Errorf("error in generating server private key: %v", err) return err } + glog.Info("success: server private key created") // sign the server cert serverCertBytes, err := x509.CreateCertificate(cryptorand.Reader, certConf, caConf, &serverPrivKey.PublicKey, caPrivKey) @@ -115,6 +118,7 @@ func InitWebhookServer() error { glog.Errorf("error in generating server certificate: %v", err) return err } + glog.Info("success: server certificate created") // PEM encode the server cert and key serverCertPEM = new(bytes.Buffer) @@ -122,12 +126,14 @@ func InitWebhookServer() error { Type: "CERTIFICATE", Bytes: serverCertBytes, }) + glog.Info("success: server certificate encoded to pem format") serverPrivKeyPEM = new(bytes.Buffer) _ = pem.Encode(serverPrivKeyPEM, &pem.Block{ Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(serverPrivKey), }) + glog.Info("success: server private key encoded to pem format") err = os.MkdirAll(certKeyPath, 0666) if err != nil { @@ -139,17 +145,20 @@ func InitWebhookServer() error { glog.Errorf("error in writing tls.crt: %v", err) return err } + glog.Infof("success: server cert (tls.crt) copied to %s", certKeyPath) err = writeFile(certKeyPath+"tls.key", serverPrivKeyPEM) if err != nil { glog.Errorf("error in writing tls.key: %v", err) return err } + glog.Infof("success: server key (tls.key) copied to %s", certKeyPath) err = patchValidatingWebhookConfig(caPEM, validatingWebhookConfig) if err != nil { return err } + glog.Infof("success: validatingwebhookconfiguration %s patched", validatingWebhookConfig) return nil } @@ -193,13 +202,21 @@ func patchValidatingWebhookConfig(caPEM *bytes.Buffer, validatingWebhookConfig s type patchStringValue struct { Op string `json:"op"` Path string `json:"path"` - Value string `json:"value"` - } + Value []byte `json:"value"` + } + /* + payload := []patchStringValue{{ + Op: "replace", + Path: "/webhooks/0/clientConfig/caBundle", + Value: base64.StdEncoding.EncodeToString(caPEM.Bytes()), + }} + */ payload := []patchStringValue{{ Op: "replace", Path: "/webhooks/0/clientConfig/caBundle", - Value: base64.StdEncoding.EncodeToString(caPEM.Bytes()), + Value: caPEM.Bytes(), }} + payloadBytes, _ := json.Marshal(payload) _, err = kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Patch( diff --git a/deploy/kubefledged-deployment-webhook-server.yaml b/deploy/kubefledged-deployment-webhook-server.yaml index d585862b..368aa956 100644 --- a/deploy/kubefledged-deployment-webhook-server.yaml +++ b/deploy/kubefledged-deployment-webhook-server.yaml @@ -37,16 +37,15 @@ spec: - name: CERT_KEY_PATH value: "/var/run/secrets/webhook-server/" volumeMounts: - - name: cert-key-volume + - name: certkey-volume mountPath: "/var/run/secrets/webhook-server" - readOnly: true containers: - image: senthilrch/kubefledged-webhook-server:v0.8.1 command: ["/opt/bin/kubefledged-webhook-server"] args: - "--stderrthreshold=INFO" - - "--cert-file=/var/run/secrets/webhook-server/cert.pem" - - "--key-file=/var/run/secrets/webhook-server/key.pem" + - "--cert-file=/var/run/secrets/webhook-server/tls.crt" + - "--key-file=/var/run/secrets/webhook-server/tls.key" - "--port=443" imagePullPolicy: Always name: webhook-server @@ -56,9 +55,10 @@ spec: fieldRef: fieldPath: metadata.namespace volumeMounts: - - name: cert-key-volume + - name: certkey-volume mountPath: "/var/run/secrets/webhook-server" readOnly: true volumes: - - name: cert-key-volume + - name: certkey-volume emptyDir: {} + serviceAccountName: kubefledged-webhook-server diff --git a/deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml b/deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml index fffdea3b..92f1fc55 100644 --- a/deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml +++ b/deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml @@ -1,7 +1,9 @@ -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: kubefledgeds.charts.helm.k8s.io + annotations: + api-approved.kubernetes.io: "https://github.com/kubernetes/enhancements/pull/1111" spec: group: charts.helm.k8s.io names: @@ -10,13 +12,11 @@ spec: plural: kubefledgeds singular: kubefledged scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - type: object - x-kubernetes-preserve-unknown-fields: true versions: - name: v1alpha2 served: true storage: true + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true \ No newline at end of file diff --git a/deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha2_kubefledged_cr.yaml b/deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha2_kubefledged_cr.yaml index 13c1bcb1..4ec04d5c 100644 --- a/deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha2_kubefledged_cr.yaml +++ b/deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha2_kubefledged_cr.yaml @@ -2,8 +2,7 @@ apiVersion: charts.helm.k8s.io/v1alpha2 kind: KubeFledged metadata: name: kube-fledged - namespace: {{OPERATOR_NAMESPACE}} + namespace: kubefledged-operator spec: # Defaults defined in /helm-charts/kubefledged/values.yaml - kubefledgedNameSpace: {{KUBEFLEDGED_NAMESPACE}} - validatingWebhookCABundle: {{CA_BUNDLE}} + kubefledgedNameSpace: kube-fledged diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/clusterrole.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/clusterrole-controller.yaml similarity index 100% rename from deploy/kubefledged-operator/helm-charts/kubefledged/templates/clusterrole.yaml rename to deploy/kubefledged-operator/helm-charts/kubefledged/templates/clusterrole-controller.yaml diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/clusterrole-webhook-server.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/clusterrole-webhook-server.yaml new file mode 100644 index 00000000..2e4c63ab --- /dev/null +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/clusterrole-webhook-server.yaml @@ -0,0 +1,18 @@ +{{- if .Values.clusterRole.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "kubefledged.fullname" . }}-webhook-server + labels: + {{ include "kubefledged.labels" . | nindent 4 }} + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" +rules: + - apiGroups: + - "admissionregistration.k8s.io" + resources: + - validatingwebhookconfigurations + verbs: + - get + - patch +{{- end -}} \ No newline at end of file diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/clusterrolebinding.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/clusterrolebinding-controller.yaml similarity index 100% rename from deploy/kubefledged-operator/helm-charts/kubefledged/templates/clusterrolebinding.yaml rename to deploy/kubefledged-operator/helm-charts/kubefledged/templates/clusterrolebinding-controller.yaml diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/clusterrolebinding-webhook-server.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/clusterrolebinding-webhook-server.yaml new file mode 100644 index 00000000..1b384df1 --- /dev/null +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/clusterrolebinding-webhook-server.yaml @@ -0,0 +1,19 @@ +{{- if .Values.clusterRoleBinding.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "kubefledged.fullname" . }}-webhook-server + labels: + {{ include "kubefledged.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "kubefledged.fullname" . }}-webhook-server +subjects: +- kind: ServiceAccount + name: {{ include "kubefledged.fullname" . }}-webhook-server + namespace: {{ .Values.kubefledgedNameSpace }} +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes +{{- end -}} diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-controller.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-controller.yaml index 5618588b..ec756ecd 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-controller.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-controller.yaml @@ -35,7 +35,7 @@ spec: - "60" - "--retry-delay" - "1" - - "https://{{ include "kubefledged.fullname" . }}-webhook-server:{{ .Values.webhookService.port }}/readyz" + - "https://{{ include "kubefledged.webhookServiceName" . }}:{{ .Values.webhookService.port }}/readyz" imagePullPolicy: {{ .Values.image.pullPolicy }} containers: - name: {{ .Chart.Name }} diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-webhook-server.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-webhook-server.yaml index 7a34102a..a76f1448 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-webhook-server.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-webhook-server.yaml @@ -19,9 +19,31 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ include "kubefledged.serviceAccountName" . }} + serviceAccountName: {{ include "kubefledged.fullname" . }}-webhook-server securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + initContainers: + - image: {{ .Values.image.kubefledgedWebhookServerRepository }}:{{ .Chart.AppVersion }} + command: {{ .Values.command.kubefledgedWebhookServerCommand }} + args: + - "--stderrthreshold=INFO" + - "--init-server" + imagePullPolicy: Always + name: init + env: + - name: KUBEFLEDGED_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: WEBHOOK_SERVER_SERVICE + value: {{ include "kubefledged.webhookServiceName" . }} + - name: VALIDATING_WEBHOOK_CONFIG + value: {{ include "kubefledged.validatingWebhookName" . }} + - name: CERT_KEY_PATH + value: "/var/run/secrets/webhook-server/" + volumeMounts: + - name: certkey-volume + mountPath: "/var/run/secrets/webhook-server" containers: - name: {{ .Chart.Name }} securityContext: @@ -42,13 +64,12 @@ spec: resources: {{- toYaml .Values.resources | nindent 12 }} volumeMounts: - - name: secret-volume + - name: certkey-volume mountPath: "/var/run/secrets/webhook-server" readOnly: true volumes: - - name: secret-volume - secret: - secretName: kubefledged-webhook-server + - name: certkey-volume + emptyDir: {} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/serviceaccount.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/serviceaccount-controller.yaml similarity index 100% rename from deploy/kubefledged-operator/helm-charts/kubefledged/templates/serviceaccount.yaml rename to deploy/kubefledged-operator/helm-charts/kubefledged/templates/serviceaccount-controller.yaml diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/serviceaccount-webhook-server.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/serviceaccount-webhook-server.yaml new file mode 100644 index 00000000..1552c565 --- /dev/null +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/serviceaccount-webhook-server.yaml @@ -0,0 +1,9 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kubefledged.fullname" . }}-webhook-server + labels: + {{ include "kubefledged.labels" . | nindent 4 }} + namespace: {{ .Values.kubefledgedNameSpace }} +{{- end -}} diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml index 2b66e6d6..c9c12366 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml @@ -15,7 +15,7 @@ webhooks: name: {{ include "kubefledged.webhookServiceName" . }} path: "/validate-image-cache" port: {{ .Values.webhookService.port }} - caBundle: {{ .Values.validatingWebhookCABundle }} + caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZCVENDQXUyZ0F3SUJBZ0lDQitVd0RRWUpLb1pJaHZjTkFRRUxCUUF3R1RFWE1CVUdBMVVFQ2hNT2EzVmkKWldac1pXUm5aV1F1YVc4d0hoY05NakV3TnpJeU1EZ3hPVFEwV2hjTk1qSXdOekl5TURneE9UUTBXakFaTVJjdwpGUVlEVlFRS0V3NXJkV0psWm14bFpHZGxaQzVwYnpDQ0FpSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnSVBBRENDCkFnb0NnZ0lCQU93dDNjWm12SnRNN1NKUGx4QlRHUGtSY3lxZGlzUXNEKzRBbTdYYjVnOVhZd3FyanZwanZVdkMKTS9FVkZiK1p1aGd5R2I5b3dEb0IxOFJ1VGd2WFMrNTZ4VlNQYTE0WENoTi92U2ZZaTAydzhaZGcxdy8wNTBFRwplMXQ4ZytTL2hXZlhxUnFORnRONTk0N25jNFcxUllJYUN5WVhjc3dGbHNMdG9xRU95aFR3ZmhyTURRY1lvaDFvCmVZRmZ1bHdiSGltdlJKYlR0QXh2b2o3OVl5MHEzTVdGWXArZ3JvR1dadk1ZeFRRSjZKT0F6bjdIUEFqY3ZqdjIKRmdRSTBVNDlDcUdpUzZWR0ZlOHFBSm15b3BYcHBJZ2l3ODUrbHBnYVA1Y3p6UjVjWHp6anBUczl1T2pWdllzLwpZSm5JS05nUlNJWnJkaE9oUzdJcllhM1JhTU5la3NSOWsrMm5LYXdwSkJBVy91VmszRmcrZFFWWHk2YTE0Yk5ZCmxnTis3SXptd21QTk1BVGVVOTZ5UWVrQ1R3UUlGWVkwZmlxd0ZjamlzZlJ6U1FHY1dUbk91bkV0MWlKbWlXckkKZzUvWEI5ZDhHQ3FGWkJEdnpNUjU5S1RwRlhacjNPYmxONkFIQ3VQa0xKNGZPMklDZWdoeGQ4TUsrTExkaERLMwo3N01qV1dXMkV4L1RDT2pQbUNIalFzWjNCeTVFR2hFTGdIczV6NGxTQSsrZGRFR1AvamptL3FQOUJWVmFBNXJJCkRuSCs1bHNuZkNEQXJYaE5HckVhaVA4a3JjVmF1SlRQNXdJMElORFFoeE1XMUFqdHA0SmgxSVZ1bHNuZU9CME4KempGMVIvempCbDhUTlN3Y2RnN3dCbm1lVWhDbGVvZ1MwZ1J0ejREZDEzeGwrOFc0ZGNDL0FnTUJBQUdqVnpCVgpNQTRHQTFVZER3RUIvd1FFQXdJQ2hEQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBVEFQQmdOVkhSTUJBZjhFCkJUQURBUUgvTUIwR0ExVWREZ1FXQkJRUmo4WUFSS09Vb213Z0Z6WGNrandBRFpEU2VqQU5CZ2txaGtpRzl3MEIKQVFzRkFBT0NBZ0VBQ0RRb1JFbllsd2pOb0tFTUlqUGJ5Wk1MdUQ5Smt6ZFhldzlESTBCNUtTaHkySFZBc0E2MgpiYzFZM25lYXRyQWcyUXJ5dklhYzZCUVhQbW4zUmF2V084blNuQnJDbHJkYk8vSXc2RnFtZVBVaDZSQVQzNyt6CitoWVdpL3JwL1U5bVBidm4yc2xOMVRlK3R6a1BsN01KeGxwMXRSWTU0RjB6Q0ZOdnFwUXBMUDdiS1VTVVVITmQKUW1WcWVQaHBucC9XV1dQNklXNVJ2VE8rSmhhTFpSV3hNaUxiWWtxN1lOZkI4SHhCam92T1NDMEE4TVRCRGVuTgpaV2lWcjN2K1kyOW9qZFY5R29VSzNPaDN0YVZNbStXWUxBdGxOcmpVdGxzU0NWN1dGbFhpNTZVd2xPZXd4ZGhmClcvMGdrTFkyaDNKNHdHUDZ6c09XbGgzVlVMV0w3WUZUYWllTEhpT0N3VzVaZ1FoWVRFNFAvcWdaQkZVVXRqNGUKbGRXeVFZWlBUR2dNelVxdm0wM3NDRHByRTM1eStSY0hFcEpwU2NXcy9yeW5VaXVJYnVGU3dhZUY2RktFZG5hSwpIMnRvdnpjMlBvMDJyWVFFOVhDNHdKUFpSaEpFYldocVROZ3JuL2NPRGZuNFovUVRyMGoxbGU1V3BacXE4Y1hWCjl5UHNVclVRL1g1WDNsMURtejlMcXhDd1ErZG5YU2xxczdRYnY5dDhPbUNZUEdnQVJaRU1qejFDUmJIbDZTaGkKaVAzY1JUTVRFV2hUMTJRZGZPd3djYUVCanNoQ0doVDAwZ3lUdFFzNm9wSzZLQm11RXF0cXV1TlFyMUdmaTl5bQp5WW1pNGg4RFdIdERpTEFrQW1DZWZuQXZoTWpiRXF3SzN6bmROdW1GTTAvbEtyZTBtekgvU3ZJPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== rules: - operations: ["CREATE", "UPDATE"] apiGroups: ["kubefledged.io"] diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/values.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/values.yaml index 98346ce5..477e40c9 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/values.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/values.yaml @@ -19,8 +19,8 @@ args: controllerImageCacheRefreshFrequency: 15m controllerImagePullPolicy: IfNotPresent webhookServerLogLevel: INFO - webhookServerCertFile: /var/run/secrets/webhook-server/cert.pem - webhookServerKeyFile: /var/run/secrets/webhook-server/key.pem + webhookServerCertFile: /var/run/secrets/webhook-server/tls.crt + webhookServerKeyFile: /var/run/secrets/webhook-server/tls.key webhookServerPort: 443 validatingWebhookCABundle: imagePullSecrets: [] diff --git a/deploy/kubefledged-validatingwebhook.yaml b/deploy/kubefledged-validatingwebhook.yaml index 65d6fa52..67698eef 100644 --- a/deploy/kubefledged-validatingwebhook.yaml +++ b/deploy/kubefledged-validatingwebhook.yaml @@ -17,7 +17,7 @@ webhooks: name: kubefledged-webhook-server path: "/validate-image-cache" port: 3443 - caBundle: {{CA_BUNDLE}} + caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZCVENDQXUyZ0F3SUJBZ0lDQitVd0RRWUpLb1pJaHZjTkFRRUxCUUF3R1RFWE1CVUdBMVVFQ2hNT2EzVmkKWldac1pXUm5aV1F1YVc4d0hoY05NakV3TnpJeU1EZ3hPVFEwV2hjTk1qSXdOekl5TURneE9UUTBXakFaTVJjdwpGUVlEVlFRS0V3NXJkV0psWm14bFpHZGxaQzVwYnpDQ0FpSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnSVBBRENDCkFnb0NnZ0lCQU93dDNjWm12SnRNN1NKUGx4QlRHUGtSY3lxZGlzUXNEKzRBbTdYYjVnOVhZd3FyanZwanZVdkMKTS9FVkZiK1p1aGd5R2I5b3dEb0IxOFJ1VGd2WFMrNTZ4VlNQYTE0WENoTi92U2ZZaTAydzhaZGcxdy8wNTBFRwplMXQ4ZytTL2hXZlhxUnFORnRONTk0N25jNFcxUllJYUN5WVhjc3dGbHNMdG9xRU95aFR3ZmhyTURRY1lvaDFvCmVZRmZ1bHdiSGltdlJKYlR0QXh2b2o3OVl5MHEzTVdGWXArZ3JvR1dadk1ZeFRRSjZKT0F6bjdIUEFqY3ZqdjIKRmdRSTBVNDlDcUdpUzZWR0ZlOHFBSm15b3BYcHBJZ2l3ODUrbHBnYVA1Y3p6UjVjWHp6anBUczl1T2pWdllzLwpZSm5JS05nUlNJWnJkaE9oUzdJcllhM1JhTU5la3NSOWsrMm5LYXdwSkJBVy91VmszRmcrZFFWWHk2YTE0Yk5ZCmxnTis3SXptd21QTk1BVGVVOTZ5UWVrQ1R3UUlGWVkwZmlxd0ZjamlzZlJ6U1FHY1dUbk91bkV0MWlKbWlXckkKZzUvWEI5ZDhHQ3FGWkJEdnpNUjU5S1RwRlhacjNPYmxONkFIQ3VQa0xKNGZPMklDZWdoeGQ4TUsrTExkaERLMwo3N01qV1dXMkV4L1RDT2pQbUNIalFzWjNCeTVFR2hFTGdIczV6NGxTQSsrZGRFR1AvamptL3FQOUJWVmFBNXJJCkRuSCs1bHNuZkNEQXJYaE5HckVhaVA4a3JjVmF1SlRQNXdJMElORFFoeE1XMUFqdHA0SmgxSVZ1bHNuZU9CME4KempGMVIvempCbDhUTlN3Y2RnN3dCbm1lVWhDbGVvZ1MwZ1J0ejREZDEzeGwrOFc0ZGNDL0FnTUJBQUdqVnpCVgpNQTRHQTFVZER3RUIvd1FFQXdJQ2hEQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBVEFQQmdOVkhSTUJBZjhFCkJUQURBUUgvTUIwR0ExVWREZ1FXQkJRUmo4WUFSS09Vb213Z0Z6WGNrandBRFpEU2VqQU5CZ2txaGtpRzl3MEIKQVFzRkFBT0NBZ0VBQ0RRb1JFbllsd2pOb0tFTUlqUGJ5Wk1MdUQ5Smt6ZFhldzlESTBCNUtTaHkySFZBc0E2MgpiYzFZM25lYXRyQWcyUXJ5dklhYzZCUVhQbW4zUmF2V084blNuQnJDbHJkYk8vSXc2RnFtZVBVaDZSQVQzNyt6CitoWVdpL3JwL1U5bVBidm4yc2xOMVRlK3R6a1BsN01KeGxwMXRSWTU0RjB6Q0ZOdnFwUXBMUDdiS1VTVVVITmQKUW1WcWVQaHBucC9XV1dQNklXNVJ2VE8rSmhhTFpSV3hNaUxiWWtxN1lOZkI4SHhCam92T1NDMEE4TVRCRGVuTgpaV2lWcjN2K1kyOW9qZFY5R29VSzNPaDN0YVZNbStXWUxBdGxOcmpVdGxzU0NWN1dGbFhpNTZVd2xPZXd4ZGhmClcvMGdrTFkyaDNKNHdHUDZ6c09XbGgzVlVMV0w3WUZUYWllTEhpT0N3VzVaZ1FoWVRFNFAvcWdaQkZVVXRqNGUKbGRXeVFZWlBUR2dNelVxdm0wM3NDRHByRTM1eStSY0hFcEpwU2NXcy9yeW5VaXVJYnVGU3dhZUY2RktFZG5hSwpIMnRvdnpjMlBvMDJyWVFFOVhDNHdKUFpSaEpFYldocVROZ3JuL2NPRGZuNFovUVRyMGoxbGU1V3BacXE4Y1hWCjl5UHNVclVRL1g1WDNsMURtejlMcXhDd1ErZG5YU2xxczdRYnY5dDhPbUNZUEdnQVJaRU1qejFDUmJIbDZTaGkKaVAzY1JUTVRFV2hUMTJRZGZPd3djYUVCanNoQ0doVDAwZ3lUdFFzNm9wSzZLQm11RXF0cXV1TlFyMUdmaTl5bQp5WW1pNGg4RFdIdERpTEFrQW1DZWZuQXZoTWpiRXF3SzN6bmROdW1GTTAvbEtyZTBtekgvU3ZJPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== rules: - operations: ["CREATE", "UPDATE"] apiGroups: ["kubefledged.io"] From e8c632b3bfd1378f918589e1b3c278a042d05dab Mon Sep 17 00:00:00 2001 From: Senthil Raja Chermapandian Date: Fri, 23 Jul 2021 17:22:31 +0530 Subject: [PATCH 06/19] updated makefile and manifests --- Makefile | 4 ++-- deploy/kubefledged-deployment-controller.yaml | 13 +++++++++++++ deploy/kubefledged-imagecache.yaml | 12 ++++++------ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 73b323ae..656a6876 100644 --- a/Makefile +++ b/Makefile @@ -197,11 +197,12 @@ deploy-using-yaml: kubectl apply -f deploy/kubefledged-serviceaccount.yaml kubectl apply -f deploy/kubefledged-clusterrole.yaml kubectl apply -f deploy/kubefledged-clusterrolebinding.yaml + -kubectl delete validatingwebhookconfigurations -l app=kubefledged,component=kubefledged-webhook-server kubectl apply -f deploy/kubefledged-validatingwebhook.yaml kubectl apply -f deploy/kubefledged-deployment-webhook-server.yaml - kubectl rollout status deployment kubefledged-webhook-server -n kube-fledged --watch kubectl apply -f deploy/kubefledged-service-webhook-server.yaml kubectl apply -f deploy/kubefledged-deployment-controller.yaml + kubectl rollout status deployment kubefledged-webhook-server -n kube-fledged --watch kubectl rollout status deployment kubefledged-controller -n kube-fledged --watch deploy-using-operator: @@ -233,7 +234,6 @@ remove-kubefledged: -kubectl delete -f deploy/kubefledged-namespace.yaml -kubectl delete -f deploy/kubefledged-clusterrolebinding.yaml -kubectl delete -f deploy/kubefledged-clusterrole.yaml - -kubectl delete -f deploy/kubefledged-serviceaccount.yaml -kubectl delete -f deploy/kubefledged-crd.yaml -kubectl delete -f deploy/kubefledged-validatingwebhook.yaml diff --git a/deploy/kubefledged-deployment-controller.yaml b/deploy/kubefledged-deployment-controller.yaml index a3afee6e..323a0543 100644 --- a/deploy/kubefledged-deployment-controller.yaml +++ b/deploy/kubefledged-deployment-controller.yaml @@ -17,6 +17,19 @@ spec: kubefledged: kubefledged-controller app: kubefledged spec: + initContainers: + - name: wait + image: senthilrch/kubefledged-cri-client:v0.8.1 + command: ["curl"] + args: + - "--insecure" + - "--retry-all-errors" + - "--retry" + - "60" + - "--retry-delay" + - "1" + - "https://kubefledged-webhook-server.kube-fledged.svc:3443/readyz" + imagePullPolicy: Always containers: - image: senthilrch/kubefledged-controller:v0.8.1 command: ["/opt/bin/kubefledged-controller"] diff --git a/deploy/kubefledged-imagecache.yaml b/deploy/kubefledged-imagecache.yaml index fcc2fc9e..083385e5 100644 --- a/deploy/kubefledged-imagecache.yaml +++ b/deploy/kubefledged-imagecache.yaml @@ -12,14 +12,14 @@ metadata: spec: # The "cacheSpec" field allows a user to define a list of images and onto which worker nodes those images should be cached (i.e. pre-pulled). cacheSpec: - # Specifies a list of images (nginx:1.21.0 and tomcat:10.0.6) with no node selector, hence these images will be cached in all the nodes in the cluster + # Specifies a list of images (nginx:1.21.1 and tomcat:10.0.8) with no node selector, hence these images will be cached in all the nodes in the cluster - images: - - nginx:1.21.0 - - tomcat:10.0.6 - # Specifies a list of images (redis:6.2.4 and postgres:13.3) with a node selector, hence these images will be cached only on the nodes selected by the node selector + - quay.io/bitnami/nginx:1.21.1 + - quay.io/bitnami/tomcat:10.0.8 + # Specifies a list of images (redis:6.2.5 and mariadb:10.5.11) with a node selector, hence these images will be cached only on the nodes selected by the node selector - images: - - redis:6.2.4 - - postgres:13.3 + - quay.io/bitnami/redis:6.2.5 + - quay.io/bitnami/mariadb:10.5.11 nodeSelector: tier: backend # Specifies a list of image pull secrets to pull images from private repositories into the cache From c40b099d6fb25a8843e39092d5a2a0459c576f5e Mon Sep 17 00:00:00 2001 From: Senthil Raja Chermapandian Date: Fri, 23 Jul 2021 20:47:33 +0530 Subject: [PATCH 07/19] updated helm operator --- Makefile | 14 +++--- deploy/kubefledged-operator/build/Dockerfile | 4 +- .../deploy/clusterrole.yaml | 2 +- .../charts.helm.k8s.io_kubefledgeds_crd.yaml | 22 ---------- ....helm.kubefledged.io_kubefledgeds_crd.yaml | 43 +++++++++++++++++++ ...befledged.io_v1alpha2_kubefledged_cr.yaml} | 2 +- deploy/kubefledged-operator/watches.yaml | 3 +- 7 files changed, 56 insertions(+), 34 deletions(-) delete mode 100644 deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml create mode 100644 deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_kubefledgeds_crd.yaml rename deploy/kubefledged-operator/deploy/crds/{charts.helm.k8s.io_v1alpha2_kubefledged_cr.yaml => charts.helm.kubefledged.io_v1alpha2_kubefledged_cr.yaml} (80%) diff --git a/Makefile b/Makefile index 656a6876..6e517775 100644 --- a/Makefile +++ b/Makefile @@ -63,7 +63,7 @@ ifndef ALPINE_VERSION endif ifndef OPERATORSDK_VERSION - OPERATORSDK_VERSION=v1.7.2 + OPERATORSDK_VERSION=v1.9.0 endif ifndef TARGET_PLATFORMS @@ -213,15 +213,15 @@ deploy-using-operator: sed -i "s|{{OPERATOR_NAMESPACE}}|${OPERATOR_NAMESPACE}|g" deploy/kubefledged-operator/deploy/service_account.yaml sed -i "s|{{OPERATOR_NAMESPACE}}|${OPERATOR_NAMESPACE}|g" deploy/kubefledged-operator/deploy/clusterrole_binding.yaml sed -i "s|{{OPERATOR_NAMESPACE}}|${OPERATOR_NAMESPACE}|g" deploy/kubefledged-operator/deploy/operator.yaml - kubectl apply -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml + kubectl apply -f deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_kubefledgeds_crd.yaml kubectl apply -f deploy/kubefledged-operator/deploy/service_account.yaml kubectl apply -f deploy/kubefledged-operator/deploy/clusterrole.yaml kubectl apply -f deploy/kubefledged-operator/deploy/clusterrole_binding.yaml kubectl apply -f deploy/kubefledged-operator/deploy/operator.yaml # Deploy kube-fledged to a separate namespace - sed -i "s|{{OPERATOR_NAMESPACE}}|${OPERATOR_NAMESPACE}|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha2_kubefledged_cr.yaml - sed -i "s|{{KUBEFLEDGED_NAMESPACE}}|${KUBEFLEDGED_NAMESPACE}|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha2_kubefledged_cr.yaml - kubectl apply -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha2_kubefledged_cr.yaml + sed -i "s|{{OPERATOR_NAMESPACE}}|${OPERATOR_NAMESPACE}|g" deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_v1alpha2_kubefledged_cr.yaml + sed -i "s|{{KUBEFLEDGED_NAMESPACE}}|${KUBEFLEDGED_NAMESPACE}|g" deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_v1alpha2_kubefledged_cr.yaml + kubectl apply -f deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_v1alpha2_kubefledged_cr.yaml update: kubectl scale deployment kubefledged-controller --replicas=0 -n kube-fledged @@ -239,14 +239,14 @@ remove-kubefledged: remove-operator-and-kubefledged: # Remove kubefledged and the namespace - -kubectl delete -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha2_kubefledged_cr.yaml + -kubectl delete -f deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_v1alpha2_kubefledged_cr.yaml -kubectl delete namespace ${KUBEFLEDGED_NAMESPACE} # Remove the kubefledged operator and the namespace -kubectl delete -f deploy/kubefledged-operator/deploy/operator.yaml -kubectl delete -f deploy/kubefledged-operator/deploy/clusterrole_binding.yaml -kubectl delete -f deploy/kubefledged-operator/deploy/clusterrole.yaml -kubectl delete -f deploy/kubefledged-operator/deploy/service_account.yaml - -kubectl delete -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml + -kubectl delete -f deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_kubefledgeds_crd.yaml -kubectl delete namespace ${OPERATOR_NAMESPACE} -git checkout deploy/kubefledged-operator/deploy/operator.yaml -git checkout deploy/kubefledged-operator/deploy/clusterrole_binding.yaml diff --git a/deploy/kubefledged-operator/build/Dockerfile b/deploy/kubefledged-operator/build/Dockerfile index d84a900c..b8ef57be 100644 --- a/deploy/kubefledged-operator/build/Dockerfile +++ b/deploy/kubefledged-operator/build/Dockerfile @@ -2,5 +2,7 @@ ARG OPERATORSDK_VERSION FROM quay.io/operator-framework/helm-operator:${OPERATORSDK_VERSION} +ENV HOME=/opt/helm COPY watches.yaml ${HOME}/watches.yaml -COPY helm-charts/ ${HOME}/helm-charts/ +COPY helm-charts ${HOME}/helm-charts +WORKDIR ${HOME} diff --git a/deploy/kubefledged-operator/deploy/clusterrole.yaml b/deploy/kubefledged-operator/deploy/clusterrole.yaml index c65e523a..3e1584f9 100644 --- a/deploy/kubefledged-operator/deploy/clusterrole.yaml +++ b/deploy/kubefledged-operator/deploy/clusterrole.yaml @@ -72,7 +72,7 @@ rules: verbs: - get - apiGroups: - - charts.helm.k8s.io + - charts.helm.kubefledged.io resources: - '*' verbs: diff --git a/deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml b/deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml deleted file mode 100644 index 92f1fc55..00000000 --- a/deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: kubefledgeds.charts.helm.k8s.io - annotations: - api-approved.kubernetes.io: "https://github.com/kubernetes/enhancements/pull/1111" -spec: - group: charts.helm.k8s.io - names: - kind: KubeFledged - listKind: KubeFledgedList - plural: kubefledgeds - singular: kubefledged - scope: Namespaced - versions: - - name: v1alpha2 - served: true - storage: true - schema: - openAPIV3Schema: - type: object - x-kubernetes-preserve-unknown-fields: true \ No newline at end of file diff --git a/deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_kubefledgeds_crd.yaml b/deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_kubefledgeds_crd.yaml new file mode 100644 index 00000000..6405ac94 --- /dev/null +++ b/deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_kubefledgeds_crd.yaml @@ -0,0 +1,43 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: kubefledgeds.charts.helm.kubefledged.io +spec: + group: charts.helm.kubefledged.io + names: + kind: KubeFledged + listKind: KubeFledgedList + plural: kubefledgeds + singular: kubefledged + scope: Namespaced + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: KubeFledged is the Schema for the kubefledgeds API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec defines the desired state of KubeFledged + type: object + x-kubernetes-preserve-unknown-fields: true + status: + description: Status defines the observed state of KubeFledged + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} \ No newline at end of file diff --git a/deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha2_kubefledged_cr.yaml b/deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_v1alpha2_kubefledged_cr.yaml similarity index 80% rename from deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha2_kubefledged_cr.yaml rename to deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_v1alpha2_kubefledged_cr.yaml index 4ec04d5c..31f8214e 100644 --- a/deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha2_kubefledged_cr.yaml +++ b/deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_v1alpha2_kubefledged_cr.yaml @@ -1,4 +1,4 @@ -apiVersion: charts.helm.k8s.io/v1alpha2 +apiVersion: charts.helm.kubefledged.io/v1alpha2 kind: KubeFledged metadata: name: kube-fledged diff --git a/deploy/kubefledged-operator/watches.yaml b/deploy/kubefledged-operator/watches.yaml index d09e7c35..907515cb 100644 --- a/deploy/kubefledged-operator/watches.yaml +++ b/deploy/kubefledged-operator/watches.yaml @@ -1,5 +1,4 @@ ---- - version: v1alpha2 - group: charts.helm.k8s.io + group: charts.helm.kubefledged.io kind: KubeFledged chart: helm-charts/kubefledged From 926f3c3cd60a3ae9ad48310c21df6ba68b382fe2 Mon Sep 17 00:00:00 2001 From: Senthil Raja Chermapandian Date: Mon, 26 Jul 2021 19:55:55 +0530 Subject: [PATCH 08/19] makefile wait for operator ready --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 6e517775..d0a9ca2d 100644 --- a/Makefile +++ b/Makefile @@ -221,6 +221,7 @@ deploy-using-operator: # Deploy kube-fledged to a separate namespace sed -i "s|{{OPERATOR_NAMESPACE}}|${OPERATOR_NAMESPACE}|g" deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_v1alpha2_kubefledged_cr.yaml sed -i "s|{{KUBEFLEDGED_NAMESPACE}}|${KUBEFLEDGED_NAMESPACE}|g" deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_v1alpha2_kubefledged_cr.yaml + kubectl rollout status deployment kubefledged-operator -n kubefledged-operator --watch kubectl apply -f deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_v1alpha2_kubefledged_cr.yaml update: From 14e6e0dbd3e2b00b0396e6fa2a05be901ab5d880 Mon Sep 17 00:00:00 2001 From: Senthil Raja Chermapandian Date: Mon, 26 Jul 2021 19:56:30 +0530 Subject: [PATCH 09/19] get imagecache before updating status --- cmd/controller/app/controller.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/controller/app/controller.go b/cmd/controller/app/controller.go index 66e33f85..7daa0901 100644 --- a/cmd/controller/app/controller.go +++ b/cmd/controller/app/controller.go @@ -635,10 +635,13 @@ func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error { } func (c *Controller) updateImageCacheStatus(imageCache *v1alpha2.ImageCache, status *v1alpha2.ImageCacheStatus) error { + imageCacheCopy, err := c.kubefledgedclientset.KubefledgedV1alpha2().ImageCaches(imageCache.Namespace).Get(context.TODO(), imageCache.Name, metav1.GetOptions{}) + if err != nil { + return err + } // NEVER modify objects from the store. It's a read-only, local cache. // You can use DeepCopy() to make a deep copy of original object and modify this copy // Or create a copy manually for better performance - imageCacheCopy := imageCache.DeepCopy() imageCacheCopy.Status = *status if imageCacheCopy.Status.Status != v1alpha2.ImageCacheActionStatusProcessing { completionTime := metav1.Now() @@ -648,7 +651,7 @@ func (c *Controller) updateImageCacheStatus(imageCache *v1alpha2.ImageCache, sta // we must use Update instead of UpdateStatus to update the Status block of the ImageCache resource. // UpdateStatus will not allow changes to the Spec of the resource, // which is ideal for ensuring nothing other than resource status has been updated. - _, err := c.kubefledgedclientset.KubefledgedV1alpha2().ImageCaches(imageCache.Namespace).Update(context.TODO(), imageCacheCopy, metav1.UpdateOptions{}) + _, err = c.kubefledgedclientset.KubefledgedV1alpha2().ImageCaches(imageCache.Namespace).Update(context.TODO(), imageCacheCopy, metav1.UpdateOptions{}) return err } From 8318799b8485a09c7759048194a9c64026dbf15e Mon Sep 17 00:00:00 2001 From: Senthil Raja Chermapandian Date: Mon, 26 Jul 2021 20:49:05 +0530 Subject: [PATCH 10/19] pre-install hook for validatingwebhookconfiguration --- .../helm-charts/kubefledged/templates/validatingwebhook.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml index c9c12366..0c1fe243 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml @@ -3,6 +3,8 @@ apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: name: {{ include "kubefledged.validatingWebhookName" . }} + annotations: + helm.sh/hook: "pre-install" webhooks: - name: validate-image-cache.kubefledged.io admissionReviewVersions: ["v1beta1", "v1"] From 52533640b42359a958a3a58acb9fc69e25163e38 Mon Sep 17 00:00:00 2001 From: Senthil Raja Chermapandian Date: Wed, 28 Jul 2021 16:15:10 +0530 Subject: [PATCH 11/19] fix golint errors --- cmd/webhook-server/app/init.go | 3 +++ cmd/webhook-server/app/server.go | 1 + 2 files changed, 4 insertions(+) diff --git a/cmd/webhook-server/app/init.go b/cmd/webhook-server/app/init.go index ae88adaa..19de9427 100644 --- a/cmd/webhook-server/app/init.go +++ b/cmd/webhook-server/app/init.go @@ -36,6 +36,9 @@ import ( "k8s.io/client-go/rest" ) +// InitWebhookServer initialises kube-fledged webhook server:- +// - generates cert/key pair +// - patched CA bundle to validatingwebhookconfiguration func InitWebhookServer() error { var caPEM, serverCertPEM, serverPrivKeyPEM *bytes.Buffer diff --git a/cmd/webhook-server/app/server.go b/cmd/webhook-server/app/server.go index 58284fb5..ab8c36f7 100644 --- a/cmd/webhook-server/app/server.go +++ b/cmd/webhook-server/app/server.go @@ -252,6 +252,7 @@ func mutateImageCache(w http.ResponseWriter, r *http.Request) { // serve(w, r, newDelegateToV1AdmitHandler(webhook.MutateImageCache)) } +// StartWebhookServer starts a new wwebhook server for kube-fledged func StartWebhookServer(certFile string, keyFile string, port int) error { config := Config{ CertFile: certFile, From ab7c2be059ea572160d9fefe6848c653f34251b5 Mon Sep 17 00:00:00 2001 From: Senthil Raja Chermapandian Date: Wed, 28 Jul 2021 16:16:11 +0530 Subject: [PATCH 12/19] modify refresh/purge annotation key to kubefledged.io/xxx --- README.md | 19 +++---------------- cmd/controller/app/controller.go | 4 ++-- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 7c263ce8..6515bb5f 100644 --- a/README.md +++ b/README.md @@ -92,26 +92,13 @@ These instructions install _kube-fledged_ to a separate namespace called "kube-f $ kubectl create namespace ${KUBEFLEDGED_NAMESPACE} ``` -- Create secret containing cert/key for kubefledged-webhook-server - - ``` - $ curl -fsSL https://raw.githubusercontent.com/senthilrch/kube-fledged/master/deploy/webhook-create-signed-cert.sh | bash -s -- --namespace ${KUBEFLEDGED_NAMESPACE} - ``` - -- Retrieve the certificate-authoity-data of the kubernetes cluster - - ``` - $ CLUSTER=$(kubectl config view --raw --flatten -o json | jq -r '.contexts[] | select(.name == "'$(kubectl config current-context)'") | .context.cluster') - $ export CA_BUNDLE=$(kubectl config view --raw --flatten -o json | jq -r '.clusters[] | select(.name == "'${CLUSTER}'") | .cluster."certificate-authority-data"') - ``` - - Verify and install latest version of kube-fledged helm chart ``` $ helm repo add kubefledged-charts https://senthilrch.github.io/kubefledged-charts/ $ gpg --keyserver keyserver.ubuntu.com --recv-keys 92D793FA3A6460ED (or) gpg --keyserver pgp.mit.edu --recv-keys 92D793FA3A6460ED $ gpg --export >~/.gnupg/pubring.gpg - $ helm install --verify kube-fledged kubefledged-charts/kube-fledged -n ${KUBEFLEDGED_NAMESPACE} --set validatingWebhookCABundle=${CA_BUNDLE} --wait + $ helm install --verify kube-fledged kubefledged-charts/kube-fledged -n ${KUBEFLEDGED_NAMESPACE} --wait ``` ## Quick Install using Helm operator @@ -257,7 +244,7 @@ $ kubectl get imagecaches imagecache1 -n kube-fledged -o json _kube-fledged_ supports both automatic and on-demand refresh of image cache. Auto refresh is enabled using the flag `--image-cache-refresh-frequency:`. To request for an on-demand refresh, run the following command:- ``` -$ kubectl annotate imagecaches imagecache1 -n kube-fledged kubefledged.k8s.io/refresh-imagecache= +$ kubectl annotate imagecaches imagecache1 -n kube-fledged kubefledged.io/refresh-imagecache= ``` ### Delete image cache @@ -265,7 +252,7 @@ $ kubectl annotate imagecaches imagecache1 -n kube-fledged kubefledged.k8s.io/re Before you could delete the image cache, you need to purge the images in the cache using the following command. This will remove all cached images from the worker nodes. ``` -$ kubectl annotate imagecaches imagecache1 -n kube-fledged kubefledged.k8s.io/purge-imagecache= +$ kubectl annotate imagecaches imagecache1 -n kube-fledged kubefledged.io/purge-imagecache= ``` View the status of purging the image cache. If any failures, such images should be removed manually or you could decide to leave the images in the worker nodes. diff --git a/cmd/controller/app/controller.go b/cmd/controller/app/controller.go index 7daa0901..fe38d36c 100644 --- a/cmd/controller/app/controller.go +++ b/cmd/controller/app/controller.go @@ -45,8 +45,8 @@ import ( ) const controllerAgentName = "kubefledged-controller" -const imageCachePurgeAnnotationKey = "kubefledged.k8s.io/purge-imagecache" -const imageCacheRefreshAnnotationKey = "kubefledged.k8s.io/refresh-imagecache" +const imageCachePurgeAnnotationKey = "kubefledged.io/purge-imagecache" +const imageCacheRefreshAnnotationKey = "kubefledged.io/refresh-imagecache" const ( // SuccessSynced is used as part of the Event 'reason' when a ImageCache is synced From c3981349cfaa33ecd673ffe645ede6ac81d9b185 Mon Sep 17 00:00:00 2001 From: Senthil Raja Chermapandian Date: Thu, 29 Jul 2021 12:11:10 +0530 Subject: [PATCH 13/19] cri-client-image name as env instead of cmd flag --- README.md | 4 ---- cmd/controller/main.go | 4 +++- deploy/kubefledged-deployment-controller.yaml | 3 ++- .../kubefledged/templates/deployment-controller.yaml | 3 ++- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 6515bb5f..614661bc 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,6 @@ These instructions install _kube-fledged_ to a separate namespace called "kube-f ``` $ kubectl get pods -n kube-fledged -l app=kubefledged - $ kubectl logs -f -n kube-fledged $ kubectl get imagecaches -n kube-fledged (Output should be: 'No resources found') ``` @@ -123,7 +122,6 @@ These instructions install _kube-fledged_ to a separate namespace called "kube-f ``` $ kubectl get pods -n kube-fledged -l app.kubernetes.io/name=kube-fledged - $ kubectl logs -f -n kube-fledged $ kubectl get imagecaches -n kube-fledged (Output should be: 'No resources found') ``` @@ -291,8 +289,6 @@ For more detailed description, go through _kube-fledged's_ [design proposal](doc `--image-cache-refresh-frequency:` The image cache is refreshed periodically to ensure the cache is up to date. Setting this flag to "0s" will disable refresh. default "15m" -`--cri-client-image:` The image name of the cri client. The cri client is used when deleting images during purging the cache". - `--image-pull-policy:` Image pull policy for pulling images into and refreshing the cache. Possible values are 'IfNotPresent' and 'Always'. Default value is 'IfNotPresent'. Image with no or ":latest" tag are always pulled. `--stderrthreshold:` Log level. set the value of this flag to INFO diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 48cd17ee..751e1167 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -89,9 +89,11 @@ func main() { func init() { flag.DurationVar(&imagePullDeadlineDuration, "image-pull-deadline-duration", time.Minute*5, "Maximum duration allowed for pulling an image. After this duration, image pull is considered to have failed") flag.DurationVar(&imageCacheRefreshFrequency, "image-cache-refresh-frequency", time.Minute*15, "The image cache is refreshed periodically to ensure the cache is up to date. Setting this flag to 0s will disable refresh") - flag.StringVar(&dockerClientImage, "cri-client-image", "senthilrch/kubefledged-cri-client:latest", "The image name of the cri client. the cri client is used when deleting images during purging the cache") flag.StringVar(&imagePullPolicy, "image-pull-policy", "IfNotPresent", "Image pull policy for pulling images into the cache. Possible values are 'IfNotPresent' and 'Always'. Default value is 'IfNotPresent'. Images with no or ':latest' tag are always pulled") if fledgedNameSpace = os.Getenv("KUBEFLEDGED_NAMESPACE"); fledgedNameSpace == "" { fledgedNameSpace = "kube-fledged" } + if dockerClientImage = os.Getenv("KUBEFLEDGED_CRI_CLIENT_IMAGE"); dockerClientImage == "" { + dockerClientImage = "senthilrch/kubefledged-cri-client:latest" + } } diff --git a/deploy/kubefledged-deployment-controller.yaml b/deploy/kubefledged-deployment-controller.yaml index 323a0543..83c63f89 100644 --- a/deploy/kubefledged-deployment-controller.yaml +++ b/deploy/kubefledged-deployment-controller.yaml @@ -37,7 +37,6 @@ spec: - "--stderrthreshold=INFO" - "--image-pull-deadline-duration=5m" - "--image-cache-refresh-frequency=15m" - - "--cri-client-image=senthilrch/kubefledged-cri-client:v0.8.1" - "--image-pull-policy=IfNotPresent" imagePullPolicy: Always name: controller @@ -46,4 +45,6 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: KUBEFLEDGED_CRI_CLIENT_IMAGE + value: "senthilrch/kubefledged-cri-client:v0.8.1" serviceAccountName: kubefledged-controller diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-controller.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-controller.yaml index ec756ecd..ee175cc3 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-controller.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-controller.yaml @@ -47,7 +47,6 @@ spec: - "--stderrthreshold={{ .Values.args.controllerLogLevel }}" - "--image-pull-deadline-duration={{ .Values.args.controllerImagePullDeadlineDuration }}" - "--image-cache-refresh-frequency={{ .Values.args.controllerImageCacheRefreshFrequency }}" - - "--cri-client-image={{ .Values.image.kubefledgedCRIClientRepository }}:{{ .Chart.AppVersion }}" - "--image-pull-policy={{ .Values.args.controllerImagePullPolicy }}" imagePullPolicy: {{ .Values.image.pullPolicy }} env: @@ -55,6 +54,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: KUBEFLEDGED_CRI_CLIENT_IMAGE + value: {{ .Values.image.kubefledgedCRIClientRepository }}:{{ .Chart.AppVersion }} resources: {{- toYaml .Values.resources | nindent 12 }} {{- with .Values.nodeSelector }} From b3e09135d64b61316132eaafd0b14726235605a4 Mon Sep 17 00:00:00 2001 From: Senthil Raja Chermapandian Date: Thu, 29 Jul 2021 21:11:07 +0530 Subject: [PATCH 14/19] read busybox image from env --- cmd/controller/app/controller.go | 5 +++-- cmd/controller/app/controller_test.go | 5 +++-- cmd/controller/main.go | 12 ++++++++---- deploy/kubefledged-deployment-controller.yaml | 4 +++- .../kubefledged/templates/deployment-controller.yaml | 2 ++ .../helm-charts/kubefledged/values.yaml | 2 ++ pkg/images/image_helpers.go | 4 ++-- pkg/images/image_manager.go | 12 +++++++----- pkg/images/image_manager_test.go | 5 +++-- 9 files changed, 33 insertions(+), 18 deletions(-) diff --git a/cmd/controller/app/controller.go b/cmd/controller/app/controller.go index fe38d36c..f9f36e27 100644 --- a/cmd/controller/app/controller.go +++ b/cmd/controller/app/controller.go @@ -92,7 +92,8 @@ func NewController( imageCacheInformer informers.ImageCacheInformer, imageCacheRefreshFrequency time.Duration, imagePullDeadlineDuration time.Duration, - dockerClientImage string, + criClientImage string, + busyboxImage string, imagePullPolicy string) *Controller { runtime.Must(fledgedscheme.AddToScheme(scheme.Scheme)) @@ -116,7 +117,7 @@ func NewController( imageCacheRefreshFrequency: imageCacheRefreshFrequency, } - imageManager, _ := images.NewImageManager(controller.workqueue, controller.imageworkqueue, controller.kubeclientset, controller.fledgedNameSpace, imagePullDeadlineDuration, dockerClientImage, imagePullPolicy) + imageManager, _ := images.NewImageManager(controller.workqueue, controller.imageworkqueue, controller.kubeclientset, controller.fledgedNameSpace, imagePullDeadlineDuration, criClientImage, busyboxImage, imagePullPolicy) controller.imageManager = imageManager glog.Info("Setting up event handlers") diff --git a/cmd/controller/app/controller_test.go b/cmd/controller/app/controller_test.go index 184d8e55..d58ff227 100644 --- a/cmd/controller/app/controller_test.go +++ b/cmd/controller/app/controller_test.go @@ -60,7 +60,8 @@ func newTestController(kubeclientset kubernetes.Interface, fledgedclientset clie imagecacheInformer := fledgedInformerFactory.Kubefledged().V1alpha2().ImageCaches() imageCacheRefreshFrequency := time.Second * 0 imagePullDeadlineDuration := time.Second * 5 - dockerClientImage := "senthilrch/fledged-docker-client:latest" + criClientImage := "senthilrch/fledged-docker-client:latest" + busyboxImage := "busybox:latest" imagePullPolicy := "IfNotPresent" /* startInformers := true @@ -72,7 +73,7 @@ func newTestController(kubeclientset kubernetes.Interface, fledgedclientset clie } */ controller := NewController(kubeclientset, fledgedclientset, fledgedNameSpace, nodeInformer, imagecacheInformer, - imageCacheRefreshFrequency, imagePullDeadlineDuration, dockerClientImage, imagePullPolicy) + imageCacheRefreshFrequency, imagePullDeadlineDuration, criClientImage, busyboxImage, imagePullPolicy) controller.nodesSynced = func() bool { return true } controller.imageCachesSynced = func() bool { return true } return controller, nodeInformer, imagecacheInformer diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 751e1167..814f8755 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -37,7 +37,8 @@ import ( var ( imageCacheRefreshFrequency time.Duration imagePullDeadlineDuration time.Duration - dockerClientImage string + criClientImage string + busyboxImage string imagePullPolicy string fledgedNameSpace string webhookServerPort int @@ -70,7 +71,7 @@ func main() { controller := app.NewController(kubeClient, fledgedClient, fledgedNameSpace, kubeInformerFactory.Core().V1().Nodes(), fledgedInformerFactory.Kubefledged().V1alpha2().ImageCaches(), - imageCacheRefreshFrequency, imagePullDeadlineDuration, dockerClientImage, imagePullPolicy) + imageCacheRefreshFrequency, imagePullDeadlineDuration, criClientImage, busyboxImage, imagePullPolicy) glog.Info("Starting pre-flight checks") if err = controller.PreFlightChecks(); err != nil { @@ -93,7 +94,10 @@ func init() { if fledgedNameSpace = os.Getenv("KUBEFLEDGED_NAMESPACE"); fledgedNameSpace == "" { fledgedNameSpace = "kube-fledged" } - if dockerClientImage = os.Getenv("KUBEFLEDGED_CRI_CLIENT_IMAGE"); dockerClientImage == "" { - dockerClientImage = "senthilrch/kubefledged-cri-client:latest" + if criClientImage = os.Getenv("KUBEFLEDGED_CRI_CLIENT_IMAGE"); criClientImage == "" { + criClientImage = "senthilrch/kubefledged-cri-client:latest" + } + if busyboxImage = os.Getenv("BUSYBOX_IMAGE"); busyboxImage == "" { + busyboxImage = "busybox:1.29.2" } } diff --git a/deploy/kubefledged-deployment-controller.yaml b/deploy/kubefledged-deployment-controller.yaml index 83c63f89..bde52d55 100644 --- a/deploy/kubefledged-deployment-controller.yaml +++ b/deploy/kubefledged-deployment-controller.yaml @@ -47,4 +47,6 @@ spec: fieldPath: metadata.namespace - name: KUBEFLEDGED_CRI_CLIENT_IMAGE value: "senthilrch/kubefledged-cri-client:v0.8.1" - serviceAccountName: kubefledged-controller + - name: BUSYBOX_IMAGE + value: "busybox:1.29.2" + serviceAccountName: kubefledged-controller diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-controller.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-controller.yaml index ee175cc3..35c4a175 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-controller.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-controller.yaml @@ -56,6 +56,8 @@ spec: fieldPath: metadata.namespace - name: KUBEFLEDGED_CRI_CLIENT_IMAGE value: {{ .Values.image.kubefledgedCRIClientRepository }}:{{ .Chart.AppVersion }} + - name: BUSYBOX_IMAGE + value: {{ .Values.image.busyboxImageRepository }}:{{ .Values.image.busyboxImageVersion }} resources: {{- toYaml .Values.resources | nindent 12 }} {{- with .Values.nodeSelector }} diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/values.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/values.yaml index 477e40c9..9ea910bc 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/values.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/values.yaml @@ -8,6 +8,8 @@ kubefledgedNameSpace: kube-fledged image: kubefledgedControllerRepository: docker.io/senthilrch/kubefledged-controller kubefledgedCRIClientRepository: docker.io/senthilrch/kubefledged-cri-client + busyboxImageRepository: docker.io/busybox + busyboxImageVersion: "1.29.2" kubefledgedWebhookServerRepository: docker.io/senthilrch/kubefledged-webhook-server pullPolicy: Always command: diff --git a/pkg/images/image_helpers.go b/pkg/images/image_helpers.go index 363689e6..5b49f471 100644 --- a/pkg/images/image_helpers.go +++ b/pkg/images/image_helpers.go @@ -31,7 +31,7 @@ import ( ) // newImagePullJob constructs a job manifest for pulling an image to a node -func newImagePullJob(imagecache *fledgedv1alpha2.ImageCache, image string, node *corev1.Node, imagePullPolicy string) (*batchv1.Job, error) { +func newImagePullJob(imagecache *fledgedv1alpha2.ImageCache, image string, node *corev1.Node, imagePullPolicy string, busyboxImage string) (*batchv1.Job, error) { var pullPolicy corev1.PullPolicy = corev1.PullIfNotPresent hostname := node.Labels["kubernetes.io/hostname"] if imagecache == nil { @@ -84,7 +84,7 @@ func newImagePullJob(imagecache *fledgedv1alpha2.ImageCache, image string, node InitContainers: []corev1.Container{ { Name: "busybox", - Image: "busybox:1.29.2", + Image: busyboxImage, Command: []string{"cp", "/bin/echo", "/tmp/bin"}, VolumeMounts: []corev1.VolumeMount{ { diff --git a/pkg/images/image_manager.go b/pkg/images/image_manager.go index 700388e4..9ba300fe 100644 --- a/pkg/images/image_manager.go +++ b/pkg/images/image_manager.go @@ -66,7 +66,8 @@ type ImageManager struct { podsLister corelisters.PodLister podsSynced cache.InformerSynced imagePullDeadlineDuration time.Duration - dockerClientImage string + criClientImage string + busyboxImage string imagePullPolicy string lock sync.RWMutex } @@ -116,7 +117,7 @@ func NewImageManager( kubeclientset kubernetes.Interface, namespace string, imagePullDeadlineDuration time.Duration, - dockerClientImage, imagePullPolicy string) (*ImageManager, coreinformers.PodInformer) { + criClientImage, busyboxImage, imagePullPolicy string) (*ImageManager, coreinformers.PodInformer) { kubeInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions( kubeclientset, @@ -134,7 +135,8 @@ func NewImageManager( podsLister: podInformer.Lister(), podsSynced: podInformer.Informer().HasSynced, imagePullDeadlineDuration: imagePullDeadlineDuration, - dockerClientImage: dockerClientImage, + criClientImage: criClientImage, + busyboxImage: busyboxImage, imagePullPolicy: imagePullPolicy, } podInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -453,7 +455,7 @@ func (m *ImageManager) processNextWorkItem() bool { // pullImage pulls the image to the node func (m *ImageManager) pullImage(iwr ImageWorkRequest) (*batchv1.Job, error) { // Construct the Job manifest - newjob, err := newImagePullJob(iwr.Imagecache, iwr.Image, iwr.Node, m.imagePullPolicy) + newjob, err := newImagePullJob(iwr.Imagecache, iwr.Image, iwr.Node, m.imagePullPolicy, m.busyboxImage) if err != nil { glog.Errorf("Error when constructing job manifest: %v", err) return nil, err @@ -470,7 +472,7 @@ func (m *ImageManager) pullImage(iwr ImageWorkRequest) (*batchv1.Job, error) { // deleteImage deletes the image from the node func (m *ImageManager) deleteImage(iwr ImageWorkRequest) (*batchv1.Job, error) { // Construct the Job manifest - newjob, err := newImageDeleteJob(iwr.Imagecache, iwr.Image, iwr.Node, iwr.ContainerRuntimeVersion, m.dockerClientImage) + newjob, err := newImageDeleteJob(iwr.Imagecache, iwr.Image, iwr.Node, iwr.ContainerRuntimeVersion, m.criClientImage) if err != nil { glog.Errorf("Error when constructing job manifest: %v", err) return nil, err diff --git a/pkg/images/image_manager_test.go b/pkg/images/image_manager_test.go index 360ba3f9..dcd4f9ba 100644 --- a/pkg/images/image_manager_test.go +++ b/pkg/images/image_manager_test.go @@ -46,13 +46,14 @@ var node = corev1.Node{ func newTestImageManager(kubeclientset kubernetes.Interface, imagepullpolicy string) (*ImageManager, coreinformers.PodInformer) { imagePullDeadlineDuration := time.Millisecond * 10 - dockerClientImage := "senthilrch/fledged-docker-client:latest" + criClientImage := "senthilrch/fledged-docker-client:latest" + busyboxImage := "busybox:latest" imagePullPolicy := imagepullpolicy imagecacheworkqueue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ImageCaches") imageworkqueue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ImagePullerStatus") imagemanager, podInformer := NewImageManager(imagecacheworkqueue, imageworkqueue, kubeclientset, fledgedNameSpace, - imagePullDeadlineDuration, dockerClientImage, imagePullPolicy) + imagePullDeadlineDuration, criClientImage, busyboxImage, imagePullPolicy) imagemanager.podsSynced = func() bool { return true } return imagemanager, podInformer From d06498d453f88c4530471294586a0fe64b5a7930 Mon Sep 17 00:00:00 2001 From: Senthil Raja Chermapandian Date: Fri, 30 Jul 2021 11:40:12 +0530 Subject: [PATCH 15/19] use busybox image from gcr.io to overcome dockerhub ratelimiting --- deploy/kubefledged-deployment-controller.yaml | 4 ++-- .../kubefledged-operator/helm-charts/kubefledged/values.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/kubefledged-deployment-controller.yaml b/deploy/kubefledged-deployment-controller.yaml index bde52d55..94deab7f 100644 --- a/deploy/kubefledged-deployment-controller.yaml +++ b/deploy/kubefledged-deployment-controller.yaml @@ -48,5 +48,5 @@ spec: - name: KUBEFLEDGED_CRI_CLIENT_IMAGE value: "senthilrch/kubefledged-cri-client:v0.8.1" - name: BUSYBOX_IMAGE - value: "busybox:1.29.2" - serviceAccountName: kubefledged-controller + value: "gcr.io/google-containers/busybox:1.27.2" + serviceAccountName: kubefledged-controller diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/values.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/values.yaml index 9ea910bc..670f098c 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/values.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/values.yaml @@ -8,8 +8,8 @@ kubefledgedNameSpace: kube-fledged image: kubefledgedControllerRepository: docker.io/senthilrch/kubefledged-controller kubefledgedCRIClientRepository: docker.io/senthilrch/kubefledged-cri-client - busyboxImageRepository: docker.io/busybox - busyboxImageVersion: "1.29.2" + busyboxImageRepository: gcr.io/google-containers/busybox + busyboxImageVersion: "1.27.2" kubefledgedWebhookServerRepository: docker.io/senthilrch/kubefledged-webhook-server pullPolicy: Always command: From d8ab2e7cf90b8cd3d89b2ae09f501bb190446ac6 Mon Sep 17 00:00:00 2001 From: Senthil Raja Chermapandian Date: Fri, 30 Jul 2021 13:20:46 +0530 Subject: [PATCH 16/19] fix issue #89 change hostpath filetype to socket --- pkg/images/image_helpers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/images/image_helpers.go b/pkg/images/image_helpers.go index 5b49f471..170a7781 100644 --- a/pkg/images/image_helpers.go +++ b/pkg/images/image_helpers.go @@ -145,7 +145,7 @@ func newImageDeleteJob(imagecache *fledgedv1alpha2.ImageCache, image string, nod "controller": controllerAgentName, } - hostpathtype := corev1.HostPathFile + hostpathtype := corev1.HostPathSocket backoffLimit := int32(0) activeDeadlineSeconds := int64((time.Hour).Seconds()) From e9852dcdb1a2bd4e63fabe820b3dd80798f936b0 Mon Sep 17 00:00:00 2001 From: Senthil Raja Chermapandian Date: Fri, 30 Jul 2021 14:34:20 +0530 Subject: [PATCH 17/19] delete pre-install hook in "make remove-operator-and-kubefledged" --- Makefile | 3 ++- .../helm-charts/kubefledged/templates/_helpers.tpl | 1 + .../helm-charts/kubefledged/templates/validatingwebhook.yaml | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d0a9ca2d..2f9c55ad 100644 --- a/Makefile +++ b/Makefile @@ -197,7 +197,7 @@ deploy-using-yaml: kubectl apply -f deploy/kubefledged-serviceaccount.yaml kubectl apply -f deploy/kubefledged-clusterrole.yaml kubectl apply -f deploy/kubefledged-clusterrolebinding.yaml - -kubectl delete validatingwebhookconfigurations -l app=kubefledged,component=kubefledged-webhook-server + -kubectl delete validatingwebhookconfigurations -l app=kubefledged kubectl apply -f deploy/kubefledged-validatingwebhook.yaml kubectl apply -f deploy/kubefledged-deployment-webhook-server.yaml kubectl apply -f deploy/kubefledged-service-webhook-server.yaml @@ -241,6 +241,7 @@ remove-kubefledged: remove-operator-and-kubefledged: # Remove kubefledged and the namespace -kubectl delete -f deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_v1alpha2_kubefledged_cr.yaml + -kubectl delete validatingwebhookconfigurations -l app.kubernetes.io/name=kube-fledged -kubectl delete namespace ${KUBEFLEDGED_NAMESPACE} # Remove the kubefledged operator and the namespace -kubectl delete -f deploy/kubefledged-operator/deploy/operator.yaml diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/_helpers.tpl b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/_helpers.tpl index 262af0f8..5242d41c 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/_helpers.tpl +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/_helpers.tpl @@ -49,6 +49,7 @@ Selector labels {{- define "kubefledged.selectorLabels" -}} app.kubernetes.io/name: {{ include "kubefledged.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/part-of: {{ .Release.Name }} {{- end -}} {{/* diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml index 0c1fe243..ab373487 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml @@ -3,8 +3,10 @@ apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: name: {{ include "kubefledged.validatingWebhookName" . }} + labels: + {{ include "kubefledged.labels" . | nindent 4 }} annotations: - helm.sh/hook: "pre-install" + helm.sh/hook: "pre-install,pre-upgrade" webhooks: - name: validate-image-cache.kubefledged.io admissionReviewVersions: ["v1beta1", "v1"] From 43e84727d315400454bfcfd78ad35e6e582e702b Mon Sep 17 00:00:00 2001 From: Senthil Raja Chermapandian Date: Fri, 30 Jul 2021 17:44:00 +0530 Subject: [PATCH 18/19] add annotations to validatingwebhookconfiguration --- .../helm-charts/kubefledged/templates/validatingwebhook.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml index ab373487..a09d13a2 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml @@ -6,7 +6,10 @@ metadata: labels: {{ include "kubefledged.labels" . | nindent 4 }} annotations: - helm.sh/hook: "pre-install,pre-upgrade" + helm.sh/hook: "pre-install" + helm.sh/hook-delete-policy: "before-hook-creation" + meta.helm.sh/release-name: {{ .Release.Name }} + meta.helm.sh/release-namespace: {{ .Release.Namespace }} webhooks: - name: validate-image-cache.kubefledged.io admissionReviewVersions: ["v1beta1", "v1"] From f18cd8861efff1969346a6626e36677da68be4ca Mon Sep 17 00:00:00 2001 From: Senthil Raja Chermapandian Date: Fri, 30 Jul 2021 19:52:24 +0530 Subject: [PATCH 19/19] add "helm repo update" to readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 614661bc..54c51405 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,7 @@ These instructions install _kube-fledged_ to a separate namespace called "kube-f ``` $ helm repo add kubefledged-charts https://senthilrch.github.io/kubefledged-charts/ + $ helm repo update $ gpg --keyserver keyserver.ubuntu.com --recv-keys 92D793FA3A6460ED (or) gpg --keyserver pgp.mit.edu --recv-keys 92D793FA3A6460ED $ gpg --export >~/.gnupg/pubring.gpg $ helm install --verify kube-fledged kubefledged-charts/kube-fledged -n ${KUBEFLEDGED_NAMESPACE} --wait