From 9fadef69805d725ba8cd94d2a00bf22631a652be Mon Sep 17 00:00:00 2001 From: bhuvanessr Date: Mon, 25 May 2020 15:10:57 +0530 Subject: [PATCH 01/21] Update README --- README.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ab413442..4cffc1a9 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ of images and onto which worker nodes those images should be cached (i.e. pre-pu _kube-fledged_ provides CRUD APIs to manage the lifecycle of the image cache, and supports several configurable parameters to customize the functioning as per one's needs. ## Table of contents + @@ -52,8 +53,7 @@ _kube-fledged_ provides CRUD APIs to manage the lifecycle of the image cache, an - A functioning kubernetes cluster (v1.9 or above). It could be a simple development cluster like minikube or a large production cluster. - All master and worker nodes having the ["kubernetes.io/hostname"](https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/#kubernetes-io-hostname) label. -- Supported container runtimes: docker, containerd, cri-o -- git, make, go, docker and kubectl installed on a local linux machine. kubectl configured properly to access the cluster. +- git, make, go, docker engine (>= 19.03) and kubectl installed on a local linux machine. kubectl configured properly to access the cluster. ## Quick Install using YAML manifests @@ -70,7 +70,7 @@ These instructions install _kube-fledged_ to a separate namespace called "kube-f - Deploy _kube-fledged_ to the cluster ``` - $ make deploy + $ make deploy-using-yaml ``` - Verify if _kube-fledged_ deployed successfully @@ -133,9 +133,9 @@ These instructions will help you build _kube-fledged_ from source and deploy it ``` $ export RELEASE_VERSION= $ export FLEDGED_IMAGE_REPO=/fledged - $ export FLEDGED_DOCKER_CLIENT_IMAGE_REPO=/fledged-docker-client $ docker login -u -p - $ make fledged-image && make client-image && make push-images + $ export DOCKER_CLI_EXPERIMENTAL=enabled + $ sudo make install-buildx && sudo make fledged-image ``` ### Deploy @@ -267,6 +267,12 @@ For more detailed description, go through _kube-fledged's_ [design proposal](doc `--stderrthreshold:` Log level. set the value of this flag to INFO +## Supported Container Runtimes + +- docker +- containerd +- cri-o + ## Supported Platforms - linux/amd64 From 9221df6436eaaa5fec67784f230845a67c4a3f96 Mon Sep 17 00:00:00 2001 From: bhuvanessr Date: Mon, 25 May 2020 15:14:44 +0530 Subject: [PATCH 02/21] updated toc in README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4cffc1a9..bed3329d 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ _kube-fledged_ provides CRUD APIs to manage the lifecycle of the image cache, an - [Remove kube-fledged](#remove-kube-fledged) - [How it works](#how-it-works) - [Configuration Flags](#configuration-flags) +- [Supported Container Runtimes](#supported-container-runtimes) - [Supported Platforms](#supported-platforms) - [Built With](#built-with) - [Contributing](#contributing) From feea7740e5e6d5df110bdc505a2a5409c5117278 Mon Sep 17 00:00:00 2001 From: bhuvanessr Date: Tue, 26 May 2020 21:07:39 +0530 Subject: [PATCH 03/21] scaffold for validating webhook implementation --- cmd/fledged.go | 8 ++ pkg/webhook/mutate.go | 161 +++++++++++++++++++++++++++++ pkg/webhook/server.go | 218 ++++++++++++++++++++++++++++++++++++++++ pkg/webhook/validate.go | 102 +++++++++++++++++++ 4 files changed, 489 insertions(+) create mode 100644 pkg/webhook/mutate.go create mode 100644 pkg/webhook/server.go create mode 100644 pkg/webhook/validate.go diff --git a/cmd/fledged.go b/cmd/fledged.go index c914af95..75666658 100644 --- a/cmd/fledged.go +++ b/cmd/fledged.go @@ -32,6 +32,7 @@ import ( clientset "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned" informers "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions" "github.com/senthilrch/kube-fledged/pkg/signals" + "github.com/senthilrch/kube-fledged/pkg/webhook" ) var ( @@ -40,6 +41,7 @@ var ( dockerClientImage string imagePullPolicy string fledgedNameSpace string + webhookServerPort int ) func main() { @@ -83,6 +85,11 @@ func main() { if err = controller.Run(1, stopCh); err != nil { glog.Fatalf("Error running controller: %s", err.Error()) } + + if err := webhook.CreateAndStartWebHookServer(stopCh, webhookServerPort); err != nil { + glog.Fatalf("Error creating webhook server: %s", err.Error()) + } + } func init() { @@ -93,4 +100,5 @@ func init() { if fledgedNameSpace = os.Getenv("KUBEFLEDGED_NAMESPACE"); fledgedNameSpace == "" { fledgedNameSpace = "kube-fledged" } + flag.IntVar(&webhookServerPort, "webhook-server-port", 3443, "Webhook server port.") } diff --git a/pkg/webhook/mutate.go b/pkg/webhook/mutate.go new file mode 100644 index 00000000..2ec85664 --- /dev/null +++ b/pkg/webhook/mutate.go @@ -0,0 +1,161 @@ +/* +Copyright 2018 The kube-fledged authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "encoding/json" + "strings" + + "github.com/golang/glog" + "k8s.io/api/admission/v1beta1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// main mutation process +func (whsvr *WebhookServer) mutate(ar *v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { + req := ar.Request + var ( + availableLabels, availableAnnotations map[string]string + objectMeta *metav1.ObjectMeta + resourceNamespace, resourceName string + ) + + glog.Infof("AdmissionReview for Kind=%v, Namespace=%v Name=%v (%v) UID=%v patchOperation=%v UserInfo=%v", + req.Kind, req.Namespace, req.Name, resourceName, req.UID, req.Operation, req.UserInfo) + + switch req.Kind.Kind { + case "Deployment": + var deployment appsv1.Deployment + if err := json.Unmarshal(req.Object.Raw, &deployment); err != nil { + glog.Errorf("Could not unmarshal raw object: %v", err) + return &v1beta1.AdmissionResponse{ + Result: &metav1.Status{ + Message: err.Error(), + }, + } + } + + resourceName, resourceNamespace, objectMeta = deployment.Name, deployment.Namespace, &deployment.ObjectMeta + availableLabels = deployment.Labels + + case "Service": + var service corev1.Service + if err := json.Unmarshal(req.Object.Raw, &service); err != nil { + glog.Errorf("Could not unmarshal raw object: %v", err) + return &v1beta1.AdmissionResponse{ + Result: &metav1.Status{ + Message: err.Error(), + }, + } + } + + resourceName, resourceNamespace, objectMeta = service.Name, service.Namespace, &service.ObjectMeta + availableLabels = service.Labels + } + + if !mutationRequired(ignoredNamespaces, objectMeta) { + glog.Infof("Skipping validation for %s/%s due to policy check", resourceNamespace, resourceName) + return &v1beta1.AdmissionResponse{ + Allowed: true, + } + } + + annotations := map[string]string{admissionWebhookAnnotationStatusKey: "mutated"} + patchBytes, err := createPatch(availableAnnotations, annotations, availableLabels, addLabels) + if err != nil { + return &v1beta1.AdmissionResponse{ + Result: &metav1.Status{ + Message: err.Error(), + }, + } + } + + glog.Infof("AdmissionResponse: patch=%v\n", string(patchBytes)) + return &v1beta1.AdmissionResponse{ + Allowed: true, + Patch: patchBytes, + PatchType: func() *v1beta1.PatchType { + pt := v1beta1.PatchTypeJSONPatch + return &pt + }(), + } +} + +func mutationRequired(ignoredList []string, metadata *metav1.ObjectMeta) bool { + required := admissionRequired(ignoredList, admissionWebhookAnnotationMutateKey, metadata) + annotations := metadata.GetAnnotations() + + if annotations == nil { + annotations = map[string]string{} + } + + status := annotations[admissionWebhookAnnotationStatusKey] + if strings.ToLower(status) == "mutated" { + required = false + } + + glog.Infof("Mutation policy for %v/%v: required:%v", metadata.Namespace, metadata.Name, required) + return required +} + +func createPatch(availableAnnotations map[string]string, annotations map[string]string, availableLabels map[string]string, labels map[string]string) ([]byte, error) { + var patch []patchOperation + patch = append(patch, updateAnnotation(availableAnnotations, annotations)...) + patch = append(patch, updateLabels(availableLabels, labels)...) + return json.Marshal(patch) +} + +func updateAnnotation(target map[string]string, added map[string]string) (patch []patchOperation) { + for key, value := range added { + if target == nil || target[key] == "" { + target = map[string]string{} + patch = append(patch, patchOperation{ + Op: "add", + Path: "/metadata/annotations", + Value: map[string]string{ + key: value, + }, + }) + } else { + patch = append(patch, patchOperation{ + Op: "replace", + Path: "/metadata/annotations/" + key, + Value: value, + }) + } + } + return patch +} + +func updateLabels(target map[string]string, added map[string]string) (patch []patchOperation) { + values := make(map[string]string) + for key, value := range added { + if target == nil || target[key] == "" { + values[key] = value + } + } + + patch = append(patch, patchOperation{ + Op: "add", + Path: "/metadata/labels", + Value: values, + }) + + return patch +} diff --git a/pkg/webhook/server.go b/pkg/webhook/server.go new file mode 100644 index 00000000..84156389 --- /dev/null +++ b/pkg/webhook/server.go @@ -0,0 +1,218 @@ +/* +Copyright 2018 The kube-fledged authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strings" + + "github.com/golang/glog" + "k8s.io/api/admission/v1beta1" + admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" +) + +var ( + runtimeScheme = runtime.NewScheme() + codecs = serializer.NewCodecFactory(runtimeScheme) + deserializer = codecs.UniversalDeserializer() + // (https://github.com/kubernetes/kubernetes/issues/57982) + defaulter = runtime.ObjectDefaulter(runtimeScheme) +) + +var ( + ignoredNamespaces = []string{ + metav1.NamespaceSystem, + metav1.NamespacePublic, + } + + requiredLabels = []string{ + nameLabel, + instanceLabel, + versionLabel, + componentLabel, + partOfLabel, + managedByLabel, + } + + addLabels = map[string]string{ + nameLabel: NA, + instanceLabel: NA, + versionLabel: NA, + componentLabel: NA, + partOfLabel: NA, + managedByLabel: NA, + } +) + +const ( + admissionWebhookAnnotationValidateKey = "admission-webhook-example.banzaicloud.com/validate" + admissionWebhookAnnotationMutateKey = "admission-webhook-example.banzaicloud.com/mutate" + admissionWebhookAnnotationStatusKey = "admission-webhook-example.banzaicloud.com/status" + nameLabel = "app.kubernetes.io/name" + instanceLabel = "app.kubernetes.io/instance" + versionLabel = "app.kubernetes.io/version" + componentLabel = "app.kubernetes.io/component" + partOfLabel = "app.kubernetes.io/part-of" + managedByLabel = "app.kubernetes.io/managed-by" + NA = "not_available" +) + +type WebhookServer struct { + server *http.Server +} + +type patchOperation struct { + Op string `json:"op"` + Path string `json:"path"` + Value interface{} `json:"value,omitempty"` +} + +func init() { + _ = corev1.AddToScheme(runtimeScheme) + _ = admissionregistrationv1beta1.AddToScheme(runtimeScheme) + // defaulting with webhooks: + // https://github.com/kubernetes/kubernetes/issues/57982 + // _ = v1.AddToScheme(runtimeScheme) +} + +// CreateAndStartWebHookServer creates and starts and https webhook server +func CreateAndStartWebHookServer(stopCh <-chan struct{}, port int) error { + certFile, keyFile := "", "" + pair, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + glog.Errorf("Failed to load key pair: %v", err) + return err + } + + whsvr := &WebhookServer{ + server: &http.Server{ + Addr: fmt.Sprintf(":%v", port), + TLSConfig: &tls.Config{Certificates: []tls.Certificate{pair}}, + }, + } + + // define http server and server handler + mux := http.NewServeMux() + mux.HandleFunc("/mutate", whsvr.serve) + mux.HandleFunc("/validate", whsvr.serve) + whsvr.server.Handler = mux + + if err := whsvr.server.ListenAndServeTLS("", ""); err != nil { + glog.Errorf("Failed to listen and serve webhook server: %v", err) + return err + } + + <-stopCh + glog.Info("Shutting down webhook server") + return nil +} + +// Serve method for webhook server +func (whsvr *WebhookServer) serve(w http.ResponseWriter, r *http.Request) { + var body []byte + if r.Body != nil { + if data, err := ioutil.ReadAll(r.Body); err == nil { + body = data + } + } + + if len(body) == 0 { + glog.Error("empty body") + http.Error(w, "empty body", http.StatusBadRequest) + return + } + + // verify the content type is accurate + contentType := r.Header.Get("Content-Type") + if contentType != "application/json" { + glog.Errorf("Content-Type=%s, expect application/json", contentType) + http.Error(w, "invalid Content-Type, expect `application/json`", http.StatusUnsupportedMediaType) + return + } + + var admissionResponse *v1beta1.AdmissionResponse + ar := v1beta1.AdmissionReview{} + if _, _, err := deserializer.Decode(body, nil, &ar); err != nil { + glog.Errorf("Can't decode body: %v", err) + admissionResponse = &v1beta1.AdmissionResponse{ + Result: &metav1.Status{ + Message: err.Error(), + }, + } + } else { + fmt.Println(r.URL.Path) + if r.URL.Path == "/mutate" { + admissionResponse = whsvr.mutate(&ar) + } else if r.URL.Path == "/validate" { + admissionResponse = whsvr.validate(&ar) + } + } + + admissionReview := v1beta1.AdmissionReview{} + if admissionResponse != nil { + admissionReview.Response = admissionResponse + if ar.Request != nil { + admissionReview.Response.UID = ar.Request.UID + } + } + + resp, err := json.Marshal(admissionReview) + if err != nil { + glog.Errorf("Can't encode response: %v", err) + http.Error(w, fmt.Sprintf("could not encode response: %v", err), http.StatusInternalServerError) + } + + glog.Infof("Ready to write reponse ...") + if _, err := w.Write(resp); err != nil { + glog.Errorf("Can't write response: %v", err) + http.Error(w, fmt.Sprintf("could not write response: %v", err), http.StatusInternalServerError) + } + +} + +func admissionRequired(ignoredList []string, admissionAnnotationKey string, metadata *metav1.ObjectMeta) bool { + // skip special kubernetes system namespaces + for _, namespace := range ignoredList { + if metadata.Namespace == namespace { + glog.Infof("Skip validation for %v for it's in special namespace:%v", metadata.Name, metadata.Namespace) + return false + } + } + + annotations := metadata.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + + var required bool + switch strings.ToLower(annotations[admissionAnnotationKey]) { + default: + required = true + case "n", "no", "false", "off": + required = false + } + + return required +} diff --git a/pkg/webhook/validate.go b/pkg/webhook/validate.go new file mode 100644 index 00000000..a7a6b36a --- /dev/null +++ b/pkg/webhook/validate.go @@ -0,0 +1,102 @@ +/* +Copyright 2018 The kube-fledged authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "encoding/json" + + "github.com/golang/glog" + "k8s.io/api/admission/v1beta1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func (whsvr *WebhookServer) validate(ar *v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { + req := ar.Request + var ( + availableLabels map[string]string + objectMeta *metav1.ObjectMeta + resourceNamespace, resourceName string + ) + + glog.Infof("AdmissionReview for Kind=%v, Namespace=%v Name=%v (%v) UID=%v patchOperation=%v UserInfo=%v", + req.Kind, req.Namespace, req.Name, resourceName, req.UID, req.Operation, req.UserInfo) + + switch req.Kind.Kind { + case "Deployment": + var deployment appsv1.Deployment + if err := json.Unmarshal(req.Object.Raw, &deployment); err != nil { + glog.Errorf("Could not unmarshal raw object: %v", err) + return &v1beta1.AdmissionResponse{ + Result: &metav1.Status{ + Message: err.Error(), + }, + } + } + + resourceName, resourceNamespace, objectMeta = deployment.Name, deployment.Namespace, &deployment.ObjectMeta + availableLabels = deployment.Labels + + case "Service": + var service corev1.Service + if err := json.Unmarshal(req.Object.Raw, &service); err != nil { + glog.Errorf("Could not unmarshal raw object: %v", err) + return &v1beta1.AdmissionResponse{ + Result: &metav1.Status{ + Message: err.Error(), + }, + } + } + + resourceName, resourceNamespace, objectMeta = service.Name, service.Namespace, &service.ObjectMeta + availableLabels = service.Labels + } + + if !validationRequired(ignoredNamespaces, objectMeta) { + glog.Infof("Skipping validation for %s/%s due to policy check", resourceNamespace, resourceName) + return &v1beta1.AdmissionResponse{ + Allowed: true, + } + } + + allowed := true + var result *metav1.Status + glog.Info("available labels:", availableLabels) + glog.Info("required labels", requiredLabels) + + for _, rl := range requiredLabels { + if _, ok := availableLabels[rl]; !ok { + allowed = false + result = &metav1.Status{ + Reason: "required labels are not set", + } + break + } + } + + return &v1beta1.AdmissionResponse{ + Allowed: allowed, + Result: result, + } +} + +func validationRequired(ignoredList []string, metadata *metav1.ObjectMeta) bool { + required := admissionRequired(ignoredList, admissionWebhookAnnotationValidateKey, metadata) + glog.Infof("Validation policy for %v/%v: required:%v", metadata.Namespace, metadata.Name, required) + return required +} From 4903d6cce62e613aaafd2393b671c1f4114c6010 Mon Sep 17 00:00:00 2001 From: bhuvanessr Date: Tue, 26 May 2020 21:20:50 +0530 Subject: [PATCH 04/21] fix go-lint issues --- pkg/webhook/mutate.go | 2 +- pkg/webhook/server.go | 10 ++++++---- pkg/webhook/validate.go | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/webhook/mutate.go b/pkg/webhook/mutate.go index 2ec85664..a1ce5b2f 100644 --- a/pkg/webhook/mutate.go +++ b/pkg/webhook/mutate.go @@ -28,7 +28,7 @@ import ( ) // main mutation process -func (whsvr *WebhookServer) mutate(ar *v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { +func (whsvr *Server) mutate(ar *v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { req := ar.Request var ( availableLabels, availableAnnotations map[string]string diff --git a/pkg/webhook/server.go b/pkg/webhook/server.go index 84156389..b69d658c 100644 --- a/pkg/webhook/server.go +++ b/pkg/webhook/server.go @@ -76,10 +76,12 @@ const ( componentLabel = "app.kubernetes.io/component" partOfLabel = "app.kubernetes.io/part-of" managedByLabel = "app.kubernetes.io/managed-by" - NA = "not_available" + // NA denotes Not applicable + NA = "not_available" ) -type WebhookServer struct { +// Server denotes a webhook https server listening to validating and mutating webhooks from api-server +type Server struct { server *http.Server } @@ -106,7 +108,7 @@ func CreateAndStartWebHookServer(stopCh <-chan struct{}, port int) error { return err } - whsvr := &WebhookServer{ + whsvr := &Server{ server: &http.Server{ Addr: fmt.Sprintf(":%v", port), TLSConfig: &tls.Config{Certificates: []tls.Certificate{pair}}, @@ -130,7 +132,7 @@ func CreateAndStartWebHookServer(stopCh <-chan struct{}, port int) error { } // Serve method for webhook server -func (whsvr *WebhookServer) serve(w http.ResponseWriter, r *http.Request) { +func (whsvr *Server) serve(w http.ResponseWriter, r *http.Request) { var body []byte if r.Body != nil { if data, err := ioutil.ReadAll(r.Body); err == nil { diff --git a/pkg/webhook/validate.go b/pkg/webhook/validate.go index a7a6b36a..ef8c42bb 100644 --- a/pkg/webhook/validate.go +++ b/pkg/webhook/validate.go @@ -26,7 +26,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func (whsvr *WebhookServer) validate(ar *v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { +func (whsvr *Server) validate(ar *v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { req := ar.Request var ( availableLabels map[string]string From 52b4eb651450b9a3841fe764d41ad2f77f1a14cb Mon Sep 17 00:00:00 2001 From: bhuvanessr Date: Wed, 27 May 2020 17:37:01 +0530 Subject: [PATCH 05/21] folder structure modified to include webhook server --- build/Dockerfile.fledged | 60 +- cmd/{ => fledged}/app/controller.go | 1486 +++++------ cmd/{ => fledged}/app/controller_helpers.go | 146 +- .../app/controller_helpers_test.go | 384 +-- cmd/{ => fledged}/app/controller_test.go | 2350 ++++++++--------- cmd/{fledged.go => fledged/main.go} | 9 +- cmd/webhook-server/main.go | 286 ++ deploy/kubefledged-deployment.yaml | 68 +- deploy/kubefledged-validatingwebhook.yaml | 21 + go.mod | 1 + go.sum | 2 + pkg/webhook/imagecache.go | 113 + pkg/webhook/mutate.go | 161 -- pkg/webhook/server.go | 220 -- pkg/webhook/validate.go | 102 - 15 files changed, 2671 insertions(+), 2738 deletions(-) rename cmd/{ => fledged}/app/controller.go (97%) rename cmd/{ => fledged}/app/controller_helpers.go (96%) rename cmd/{ => fledged}/app/controller_helpers_test.go (96%) rename cmd/{ => fledged}/app/controller_test.go (96%) rename cmd/{fledged.go => fledged/main.go} (91%) create mode 100644 cmd/webhook-server/main.go create mode 100644 deploy/kubefledged-validatingwebhook.yaml create mode 100644 pkg/webhook/imagecache.go delete mode 100644 pkg/webhook/mutate.go delete mode 100644 pkg/webhook/server.go delete mode 100644 pkg/webhook/validate.go diff --git a/build/Dockerfile.fledged b/build/Dockerfile.fledged index db3b7d88..7558501d 100644 --- a/build/Dockerfile.fledged +++ b/build/Dockerfile.fledged @@ -1,30 +1,30 @@ -# Copyright 2018 The kube-fledged authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG GOLANG_VERSION -ARG ALPINE_VERSION - -FROM golang:$GOLANG_VERSION AS builder -LABEL stage=builder -ARG GIT_BRANCH -RUN mkdir -p /go/src/github.com/senthilrch && \ - git clone --depth=1 --single-branch --branch=$GIT_BRANCH https://github.com/senthilrch/kube-fledged /go/src/github.com/senthilrch/kube-fledged && \ - cd /go/src/github.com/senthilrch/kube-fledged && \ - CGO_ENABLED=0 go build -o build/fledged -ldflags '-s -w -extldflags "-static"' cmd/fledged.go - -FROM alpine:$ALPINE_VERSION -LABEL maintainer="senthilrch " -COPY --from=builder /go/src/github.com/senthilrch/kube-fledged/build/fledged /opt/bin/fledged -RUN chmod 755 /opt/bin/fledged -ENTRYPOINT ["/opt/bin/fledged"] +# Copyright 2018 The kube-fledged authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG GOLANG_VERSION +ARG ALPINE_VERSION + +FROM golang:$GOLANG_VERSION AS builder +LABEL stage=builder +ARG GIT_BRANCH +RUN mkdir -p /go/src/github.com/senthilrch && \ + git clone --depth=1 --single-branch --branch=$GIT_BRANCH https://github.com/senthilrch/kube-fledged /go/src/github.com/senthilrch/kube-fledged && \ + cd /go/src/github.com/senthilrch/kube-fledged && \ + CGO_ENABLED=0 go build -o build/fledged -ldflags '-s -w -extldflags "-static"' cmd/fledged/main.go + +FROM alpine:$ALPINE_VERSION +LABEL maintainer="senthilrch " +COPY --from=builder /go/src/github.com/senthilrch/kube-fledged/build/fledged /opt/bin/fledged +RUN chmod 755 /opt/bin/fledged +ENTRYPOINT ["/opt/bin/fledged"] diff --git a/cmd/app/controller.go b/cmd/fledged/app/controller.go similarity index 97% rename from cmd/app/controller.go rename to cmd/fledged/app/controller.go index 0299eeda..74fdf6df 100644 --- a/cmd/app/controller.go +++ b/cmd/fledged/app/controller.go @@ -1,743 +1,743 @@ -/* -Copyright 2018 The kube-fledged authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "fmt" - "reflect" - "time" - - "github.com/golang/glog" - fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" - clientset "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned" - fledgedscheme "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/scheme" - informers "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions/fledged/v1alpha1" - listers "github.com/senthilrch/kube-fledged/pkg/client/listers/fledged/v1alpha1" - "github.com/senthilrch/kube-fledged/pkg/images" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/wait" - coreinformers "k8s.io/client-go/informers/core/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/scheme" - typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" - corelisters "k8s.io/client-go/listers/core/v1" - "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/record" - "k8s.io/client-go/util/workqueue" -) - -const controllerAgentName = "fledged" -const fledgedCacheSpecValidationKey = "fledged.k8s.io/cachespecvalidation" -const imageCachePurgeAnnotationKey = "fledged.k8s.io/purge-imagecache" -const imageCacheRefreshAnnotationKey = "fledged.k8s.io/refresh-imagecache" - -const ( - // SuccessSynced is used as part of the Event 'reason' when a ImageCache is synced - SuccessSynced = "Synced" - // MessageResourceSynced is the message used for an Event fired when a ImageCache - // is synced successfully - MessageResourceSynced = "ImageCache synced successfully" -) - -// Controller is the controller for ImageCache resources -type Controller struct { - // kubeclientset is a standard kubernetes clientset - kubeclientset kubernetes.Interface - // fledgedclientset is a clientset for fledged.k8s.io API group - fledgedclientset clientset.Interface - - fledgedNameSpace string - nodesLister corelisters.NodeLister - nodesSynced cache.InformerSynced - imageCachesLister listers.ImageCacheLister - imageCachesSynced cache.InformerSynced - - // workqueue is a rate limited work queue. This is used to queue work to be - // processed instead of performing it as soon as a change happens. This - // means we can ensure we only process a fixed amount of resources at a - // time, and makes it easy to ensure we are never processing the same item - // simultaneously in two different workers. - workqueue workqueue.RateLimitingInterface - imageworkqueue workqueue.RateLimitingInterface - imageManager *images.ImageManager - // recorder is an event recorder for recording Event resources to the - // Kubernetes API. - recorder record.EventRecorder - imageCacheRefreshFrequency time.Duration -} - -// NewController returns a new fledged controller -func NewController( - kubeclientset kubernetes.Interface, - fledgedclientset clientset.Interface, - namespace string, - nodeInformer coreinformers.NodeInformer, - imageCacheInformer informers.ImageCacheInformer, - imageCacheRefreshFrequency time.Duration, - imagePullDeadlineDuration time.Duration, - dockerClientImage string, - imagePullPolicy string) *Controller { - - utilruntime.Must(fledgedscheme.AddToScheme(scheme.Scheme)) - glog.V(4).Info("Creating event broadcaster") - eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) - eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: kubeclientset.CoreV1().Events("")}) - recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerAgentName}) - - controller := &Controller{ - kubeclientset: kubeclientset, - fledgedclientset: fledgedclientset, - fledgedNameSpace: namespace, - nodesLister: nodeInformer.Lister(), - nodesSynced: nodeInformer.Informer().HasSynced, - imageCachesLister: imageCacheInformer.Lister(), - imageCachesSynced: imageCacheInformer.Informer().HasSynced, - workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ImageCaches"), - imageworkqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ImagePullerStatus"), - recorder: recorder, - imageCacheRefreshFrequency: imageCacheRefreshFrequency, - } - - imageManager, _ := images.NewImageManager(controller.workqueue, controller.imageworkqueue, controller.kubeclientset, controller.fledgedNameSpace, imagePullDeadlineDuration, dockerClientImage, imagePullPolicy) - controller.imageManager = imageManager - - glog.Info("Setting up event handlers") - // Set up an event handler for when ImageCache resources change - imageCacheInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - controller.enqueueImageCache(images.ImageCacheCreate, nil, obj) - }, - UpdateFunc: func(old, new interface{}) { - controller.enqueueImageCache(images.ImageCacheUpdate, old, new) - }, - DeleteFunc: func(obj interface{}) { - controller.enqueueImageCache(images.ImageCacheDelete, obj, nil) - }, - }) - return controller -} - -// PreFlightChecks performs pre-flight checks and actions before the controller is started -func (c *Controller) PreFlightChecks() error { - if err := c.danglingJobs(); err != nil { - return err - } - if err := c.danglingImageCaches(); err != nil { - return err - } - return nil -} - -// danglingJobs finds and removes dangling or stuck jobs -func (c *Controller) danglingJobs() error { - joblist, err := c.kubeclientset.BatchV1().Jobs(c.fledgedNameSpace).List(metav1.ListOptions{}) - if err != nil { - glog.Errorf("Error listing jobs: %v", err) - return err - } - - if joblist == nil || len(joblist.Items) == 0 { - glog.Info("No dangling or stuck jobs found...") - return nil - } - deletePropagation := metav1.DeletePropagationBackground - for _, job := range joblist.Items { - err := c.kubeclientset.BatchV1().Jobs(c.fledgedNameSpace). - Delete(job.Name, &metav1.DeleteOptions{PropagationPolicy: &deletePropagation}) - if err != nil { - glog.Errorf("Error deleting job(%s): %v", job.Name, err) - return err - } - glog.Infof("Dangling Job(%s) deleted", job.Name) - } - return nil -} - -// danglingImageCaches finds dangling or stuck image cache and marks them as abhorted. Such -// image caches will get refreshed in the next cycle -func (c *Controller) danglingImageCaches() error { - dangling := false - imagecachelist, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(c.fledgedNameSpace).List(metav1.ListOptions{}) - if err != nil { - glog.Errorf("Error listing imagecaches: %v", err) - return err - } - - if imagecachelist == nil || len(imagecachelist.Items) == 0 { - glog.Info("No dangling or stuck imagecaches found...") - return nil - } - status := &fledgedv1alpha1.ImageCacheStatus{ - Failures: map[string]fledgedv1alpha1.NodeReasonMessageList{}, - Status: fledgedv1alpha1.ImageCacheActionStatusAborted, - Reason: fledgedv1alpha1.ImageCacheReasonImagePullAborted, - Message: fledgedv1alpha1.ImageCacheMessageImagePullAborted, - } - for _, imagecache := range imagecachelist.Items { - if imagecache.Status.Status == fledgedv1alpha1.ImageCacheActionStatusProcessing { - status.StartTime = imagecache.Status.StartTime - err := c.updateImageCacheStatus(&imagecache, status) - if err != nil { - glog.Errorf("Error updating ImageCache(%s) status to '%s': %v", imagecache.Name, fledgedv1alpha1.ImageCacheActionStatusAborted, err) - return err - } - dangling = true - glog.Infof("Dangling Image cache(%s) status changed to '%s'", imagecache.Name, fledgedv1alpha1.ImageCacheActionStatusAborted) - } - } - - if !dangling { - glog.Info("No dangling or stuck imagecaches found...") - } - return nil -} - -// Run will set up the event handlers for types we are interested in, as well -// as syncing informer caches and starting workers. It will block until stopCh -// is closed, at which point it will shutdown the workqueue and wait for -// workers to finish processing their current work items. -func (c *Controller) Run(threadiness int, stopCh <-chan struct{}) error { - defer runtime.HandleCrash() - defer c.workqueue.ShutDown() - defer c.imageworkqueue.ShutDown() - - // Start the informer factories to begin populating the informer caches - glog.Info("Starting fledged controller") - - // Wait for the caches to be synced before starting workers - glog.Info("Waiting for informer caches to sync") - if ok := cache.WaitForCacheSync(stopCh, c.nodesSynced, c.imageCachesSynced); !ok { - return fmt.Errorf("failed to wait for caches to sync") - } - - glog.Info("Starting image cache worker") - // Launch workers to process ImageCache resources - for i := 0; i < threadiness; i++ { - go wait.Until(c.runWorker, time.Second, stopCh) - } - - if c.imageCacheRefreshFrequency.Nanoseconds() != int64(0) { - glog.Info("Starting cache refresh worker") - go wait.Until(c.runRefreshWorker, c.imageCacheRefreshFrequency, stopCh) - } - - glog.Info("Started workers") - c.imageManager.Run(stopCh) - if err := c.imageManager.Run(stopCh); err != nil { - glog.Fatalf("Error running image manager: %s", err.Error()) - } - - <-stopCh - glog.Info("Shutting down workers") - - return nil -} - -// enqueueImageCache takes a ImageCache resource and converts it into a namespace/name -// string which is then put onto the work queue. This method should *not* be -// passed resources of any type other than ImageCache. -func (c *Controller) enqueueImageCache(workType images.WorkType, old, new interface{}) bool { - var key string - var err error - var obj interface{} - wqKey := images.WorkQueueKey{} - - switch workType { - case images.ImageCacheCreate: - obj = new - newImageCache := new.(*fledgedv1alpha1.ImageCache) - // If the ImageCache resource already has a status field, it means it's already - // synced, so do not queue it for processing - if !reflect.DeepEqual(newImageCache.Status, fledgedv1alpha1.ImageCacheStatus{}) { - return false - } - case images.ImageCacheUpdate: - obj = new - oldImageCache := old.(*fledgedv1alpha1.ImageCache) - newImageCache := new.(*fledgedv1alpha1.ImageCache) - - if oldImageCache.Status.Status == fledgedv1alpha1.ImageCacheActionStatusProcessing { - if !reflect.DeepEqual(newImageCache.Spec, oldImageCache.Spec) { - glog.Warningf("Received image cache update/purge/delete for '%s' while it is under processing, so ignoring.", oldImageCache.Name) - return false - } - } - if _, exists := newImageCache.Annotations[imageCachePurgeAnnotationKey]; exists { - if _, exists := oldImageCache.Annotations[imageCachePurgeAnnotationKey]; !exists { - workType = images.ImageCachePurge - break - } - } - if _, exists := newImageCache.Annotations[imageCacheRefreshAnnotationKey]; exists { - if _, exists := oldImageCache.Annotations[imageCacheRefreshAnnotationKey]; !exists { - workType = images.ImageCacheRefresh - break - } - } - if !reflect.DeepEqual(newImageCache.Spec, oldImageCache.Spec) { - if validation, ok := newImageCache.Annotations[fledgedCacheSpecValidationKey]; ok { - if validation == "failed" { - if err := c.removeAnnotation(newImageCache, fledgedCacheSpecValidationKey); err != nil { - glog.Errorf("Error removing Annotation %s from imagecache(%s): %v", fledgedCacheSpecValidationKey, newImageCache.Name, err) - } - return false - } - } - } else { - return false - } - case images.ImageCacheDelete: - return false - - case images.ImageCacheRefresh: - obj = old - } - - if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { - runtime.HandleError(err) - return false - } - wqKey.WorkType = workType - wqKey.ObjKey = key - if workType == images.ImageCacheUpdate { - oldImageCache := old.(*fledgedv1alpha1.ImageCache) - wqKey.OldImageCache = oldImageCache - } - - c.workqueue.AddRateLimited(wqKey) - glog.V(4).Infof("enqueueImageCache::ImageCache resource queued for work type %s", workType) - return true -} - -// runWorker is a long-running function that will continually call the -// processNextWorkItem function in order to read and process a message on the -// workqueue. -func (c *Controller) runWorker() { - for c.processNextWorkItem() { - } -} - -// processNextWorkItem will read a single work item off the workqueue and -// attempt to process it, by calling the syncHandler. -func (c *Controller) processNextWorkItem() bool { - //glog.Info("processNextWorkItem::Beginning...") - obj, shutdown := c.workqueue.Get() - - if shutdown { - return false - } - - // We wrap this block in a func so we can defer c.workqueue.Done. - err := func(obj interface{}) error { - // We call Done here so the workqueue knows we have finished - // processing this item. We also must remember to call Forget if we - // do not want this work item being re-queued. For example, we do - // not call Forget if a transient error occurs, instead the item is - // put back on the workqueue and attempted again after a back-off - // period. - defer c.workqueue.Done(obj) - var key images.WorkQueueKey - var ok bool - // We expect strings to come off the workqueue. These are of the - // form namespace/name. We do this as the delayed nature of the - // workqueue means the items in the informer cache may actually be - // more up to date that when the item was initially put onto the - // workqueue. - if key, ok = obj.(images.WorkQueueKey); !ok { - // As the item in the workqueue is actually invalid, we call - // Forget here else we'd go into a loop of attempting to - // process a work item that is invalid. - c.workqueue.Forget(obj) - runtime.HandleError(fmt.Errorf("Unexpected type in workqueue: %#v", obj)) - return nil - } - // Run the syncHandler, passing it the namespace/name string of the - // ImageCache resource to be synced. - if err := c.syncHandler(key); err != nil { - glog.Errorf("error syncing imagecache: %v", err.Error()) - return fmt.Errorf("error syncing imagecache: %v", err.Error()) - } - // Finally, if no error occurs we Forget this item so it does not - // get queued again until another change happens. - c.workqueue.Forget(obj) - //glog.Infof("Successfully synced '%s' for event '%s'", key.ObjKey, key.WorkType) - return nil - }(obj) - - if err != nil { - runtime.HandleError(err) - return true - } - - return true -} - -// runRefreshWorker is resposible of refreshing the image cache -func (c *Controller) runRefreshWorker() { - // List the ImageCache resources - imageCaches, err := c.imageCachesLister.ImageCaches(c.fledgedNameSpace).List(labels.Everything()) - if err != nil { - glog.Errorf("Error in listing image caches: %v", err) - return - } - for i := range imageCaches { - // Do not refresh if status is not yet updated - if reflect.DeepEqual(imageCaches[i].Status, fledgedv1alpha1.ImageCacheStatus{}) { - continue - } - // Do not refresh if image cache is already under processing - if imageCaches[i].Status.Status == fledgedv1alpha1.ImageCacheActionStatusProcessing { - continue - } - // Do not refresh image cache if cache spec validation failed - if imageCaches[i].Status.Status == fledgedv1alpha1.ImageCacheActionStatusFailed && - imageCaches[i].Status.Reason == fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed { - continue - } - // Do not refresh if image cache has been purged - if imageCaches[i].Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCachePurge { - continue - } - c.enqueueImageCache(images.ImageCacheRefresh, imageCaches[i], nil) - } -} - -// syncHandler compares the actual state with the desired, and attempts to -// converge the two. It then updates the Status block of the ImageCache resource -// with the current status of the resource. -func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error { - status := &fledgedv1alpha1.ImageCacheStatus{ - Failures: map[string]fledgedv1alpha1.NodeReasonMessageList{}, - } - - // Convert the namespace/name string into a distinct namespace and name - namespace, name, err := cache.SplitMetaNamespaceKey(wqKey.ObjKey) - if err != nil { - glog.Errorf("Error from cache.SplitMetaNamespaceKey(): %v", err) - return err - } - - glog.Infof("Starting to sync image cache %s(%s)", name, wqKey.WorkType) - - switch wqKey.WorkType { - case images.ImageCacheCreate, images.ImageCacheUpdate, images.ImageCacheRefresh, images.ImageCachePurge: - - startTime := metav1.Now() - status.StartTime = &startTime - // Get the ImageCache resource with this namespace/name - imageCache, err := c.imageCachesLister.ImageCaches(namespace).Get(name) - if err != nil { - // The ImageCache resource may no longer exist, in which case we stop - // processing. - glog.Errorf("Error getting imagecache(%s): %v", name, err) - return err - } - - err = validateCacheSpec(c, imageCache) - if err != nil { - status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed - status.Reason = fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed - status.Message = err.Error() - - if err := c.updateImageCacheStatus(imageCache, status); err != nil { - glog.Errorf("Error updating imagecache status to %s: %v", status.Status, err) - return err - } - - return err - } - - if wqKey.WorkType == images.ImageCacheUpdate && wqKey.OldImageCache == nil { - status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed - status.Reason = fledgedv1alpha1.ImageCacheReasonOldImageCacheNotFound - status.Message = fledgedv1alpha1.ImageCacheMessageOldImageCacheNotFound - - if err := c.updateImageCacheStatus(imageCache, status); err != nil { - glog.Errorf("Error updating imagecache status to %s: %v", status.Status, err) - return err - } - glog.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonOldImageCacheNotFound, fledgedv1alpha1.ImageCacheMessageOldImageCacheNotFound) - return fmt.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonOldImageCacheNotFound, fledgedv1alpha1.ImageCacheMessageOldImageCacheNotFound) - } - - if wqKey.WorkType == images.ImageCacheUpdate { - if len(wqKey.OldImageCache.Spec.CacheSpec) != len(imageCache.Spec.CacheSpec) { - status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed - status.Reason = fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed - status.Message = fledgedv1alpha1.ImageCacheMessageNotSupportedUpdates - - if err = c.updateImageCacheSpecAndStatus(imageCache, wqKey.OldImageCache.Spec, status); err != nil { - glog.Errorf("Error updating imagecache spec and status to %s: %v", status.Status, err) - return err - } - glog.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, "Mismatch in no. of image lists") - return fmt.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, "Mismatch in no. of image lists") - } - - for i := range wqKey.OldImageCache.Spec.CacheSpec { - if !reflect.DeepEqual(wqKey.OldImageCache.Spec.CacheSpec[i].NodeSelector, imageCache.Spec.CacheSpec[i].NodeSelector) { - status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed - status.Reason = fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed - status.Message = fledgedv1alpha1.ImageCacheMessageNotSupportedUpdates - - if err = c.updateImageCacheSpecAndStatus(imageCache, wqKey.OldImageCache.Spec, status); err != nil { - glog.Errorf("Error updating imagecache spec and status to %s: %v", status.Status, err) - return err - } - glog.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, "Mismatch in node selector") - return fmt.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, "Mismatch in node selector") - } - } - } - - cacheSpec := imageCache.Spec.CacheSpec - glog.V(4).Infof("cacheSpec: %+v", cacheSpec) - var nodes []*corev1.Node - - status.Status = fledgedv1alpha1.ImageCacheActionStatusProcessing - - if wqKey.WorkType == images.ImageCacheCreate { - status.Reason = fledgedv1alpha1.ImageCacheReasonImageCacheCreate - status.Message = fledgedv1alpha1.ImageCacheMessagePullingImages - } - - if wqKey.WorkType == images.ImageCacheUpdate { - status.Reason = fledgedv1alpha1.ImageCacheReasonImageCacheUpdate - status.Message = fledgedv1alpha1.ImageCacheMessageUpdatingCache - } - - if wqKey.WorkType == images.ImageCacheRefresh { - status.Reason = fledgedv1alpha1.ImageCacheReasonImageCacheRefresh - status.Message = fledgedv1alpha1.ImageCacheMessageRefreshingCache - } - - if wqKey.WorkType == images.ImageCachePurge { - status.Reason = fledgedv1alpha1.ImageCacheReasonImageCachePurge - status.Message = fledgedv1alpha1.ImageCacheMessagePurgeCache - } - - imageCache, err = c.fledgedclientset.FledgedV1alpha1().ImageCaches(namespace).Get(name, metav1.GetOptions{}) - if err != nil { - glog.Errorf("Error getting imagecache(%s) from api server: %v", name, err) - return err - } - - if err = c.updateImageCacheStatus(imageCache, status); err != nil { - glog.Errorf("Error updating imagecache status to %s: %v", status.Status, err) - return err - } - - for k, i := range cacheSpec { - if len(i.NodeSelector) > 0 { - if nodes, err = c.nodesLister.List(labels.Set(i.NodeSelector).AsSelector()); err != nil { - glog.Errorf("Error listing nodes using nodeselector %+v: %v", i.NodeSelector, err) - return err - } - } else { - if nodes, err = c.nodesLister.List(labels.Everything()); err != nil { - glog.Errorf("Error listing nodes using nodeselector labels.Everything(): %v", err) - return err - } - } - glog.V(4).Infof("No. of nodes in %+v is %d", i.NodeSelector, len(nodes)) - if len(nodes) == 0 { - glog.Errorf("NodeSelector %+v did not match any nodes.", i.NodeSelector) - return fmt.Errorf("NodeSelector %+v did not match any nodes", i.NodeSelector) - } - - for _, n := range nodes { - for m := range i.Images { - ipr := images.ImageWorkRequest{ - Image: i.Images[m], - Node: n, - ContainerRuntimeVersion: n.Status.NodeInfo.ContainerRuntimeVersion, - WorkType: wqKey.WorkType, - Imagecache: imageCache, - } - c.imageworkqueue.AddRateLimited(ipr) - } - if wqKey.WorkType == images.ImageCacheUpdate { - for _, oldimage := range wqKey.OldImageCache.Spec.CacheSpec[k].Images { - matched := false - for _, newimage := range i.Images { - if oldimage == newimage { - matched = true - break - } - } - if !matched { - ipr := images.ImageWorkRequest{ - Image: oldimage, - Node: n, - ContainerRuntimeVersion: n.Status.NodeInfo.ContainerRuntimeVersion, - WorkType: images.ImageCachePurge, - Imagecache: imageCache, - } - c.imageworkqueue.AddRateLimited(ipr) - } - } - } - } - } - - // We add an empty image pull request to signal the image manager that all - // requests for this sync action have been placed in the imageworkqueue - c.imageworkqueue.AddRateLimited(images.ImageWorkRequest{WorkType: wqKey.WorkType, Imagecache: imageCache}) - - case images.ImageCacheStatusUpdate: - glog.V(4).Infof("wqKey.Status = %+v", wqKey.Status) - // Finally, we update the status block of the ImageCache resource to reflect the - // current state of the world - // Get the ImageCache resource with this namespace/name - imageCache, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(namespace).Get(name, metav1.GetOptions{}) - if err != nil { - glog.Errorf("Error getting image cache %s: %v", name, err) - return err - } - - if imageCache.Status.StartTime != nil { - status.StartTime = imageCache.Status.StartTime - } - - status.Reason = imageCache.Status.Reason - - failures := false - for _, v := range *wqKey.Status { - if (v.Status == images.ImageWorkResultStatusSucceeded || v.Status == images.ImageWorkResultStatusAlreadyPulled) && !failures { - status.Status = fledgedv1alpha1.ImageCacheActionStatusSucceeded - if v.ImageWorkRequest.WorkType == images.ImageCachePurge { - status.Message = fledgedv1alpha1.ImageCacheMessageImagesDeletedSuccessfully - } else { - status.Message = fledgedv1alpha1.ImageCacheMessageImagesPulledSuccessfully - } - } - if v.Status == images.ImageWorkResultStatusFailed && !failures { - failures = true - status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed - if v.ImageWorkRequest.WorkType == images.ImageCachePurge { - status.Message = fledgedv1alpha1.ImageCacheMessageImageDeleteFailedForSomeImages - } else { - status.Message = fledgedv1alpha1.ImageCacheMessageImagePullFailedForSomeImages - } - } - if v.Status == images.ImageWorkResultStatusFailed { - status.Failures[v.ImageWorkRequest.Image] = append( - status.Failures[v.ImageWorkRequest.Image], fledgedv1alpha1.NodeReasonMessage{ - Node: v.ImageWorkRequest.Node.Labels["kubernetes.io/hostname"], - Reason: v.Reason, - Message: v.Message, - }) - } - } - - err = c.updateImageCacheStatus(imageCache, status) - if err != nil { - glog.Errorf("Error updating ImageCache status: %v", err) - return err - } - - if imageCache.Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCachePurge || imageCache.Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCacheRefresh { - imageCache, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(namespace).Get(name, metav1.GetOptions{}) - if err != nil { - glog.Errorf("Error getting image cache %s: %v", name, err) - return err - } - if imageCache.Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCachePurge { - if err := c.removeAnnotation(imageCache, imageCachePurgeAnnotationKey); err != nil { - glog.Errorf("Error removing Annotation %s from imagecache(%s): %v", imageCachePurgeAnnotationKey, imageCache.Name, err) - return err - } - } - if imageCache.Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCacheRefresh { - if err := c.removeAnnotation(imageCache, imageCacheRefreshAnnotationKey); err != nil { - glog.Errorf("Error removing Annotation %s from imagecache(%s): %v", imageCacheRefreshAnnotationKey, imageCache.Name, err) - return err - } - } - } - - if status.Status == fledgedv1alpha1.ImageCacheActionStatusSucceeded { - c.recorder.Event(imageCache, corev1.EventTypeNormal, status.Reason, status.Message) - } - - if status.Status == fledgedv1alpha1.ImageCacheActionStatusFailed { - c.recorder.Event(imageCache, corev1.EventTypeWarning, status.Reason, status.Message) - } - } - glog.Infof("Completed sync actions for image cache %s(%s)", name, wqKey.WorkType) - return nil - -} - -func (c *Controller) updateImageCacheStatus(imageCache *fledgedv1alpha1.ImageCache, status *fledgedv1alpha1.ImageCacheStatus) error { - // NEVER modify objects from the store. It's a read-only, local cache. - // You can use DeepCopy() to make a deep copy of original object and modify this copy - // Or create a copy manually for better performance - imageCacheCopy := imageCache.DeepCopy() - imageCacheCopy.Status = *status - if imageCacheCopy.Status.Status != fledgedv1alpha1.ImageCacheActionStatusProcessing { - completionTime := metav1.Now() - imageCacheCopy.Status.CompletionTime = &completionTime - } - // If the CustomResourceSubresources feature gate is not enabled, - // we must use Update instead of UpdateStatus to update the Status block of the ImageCache resource. - // UpdateStatus will not allow changes to the Spec of the resource, - // which is ideal for ensuring nothing other than resource status has been updated. - _, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(imageCache.Namespace).Update(imageCacheCopy) - return err -} - -func (c *Controller) updateImageCacheSpecAndStatus(imageCache *fledgedv1alpha1.ImageCache, spec fledgedv1alpha1.ImageCacheSpec, status *fledgedv1alpha1.ImageCacheStatus) error { - // NEVER modify objects from the store. It's a read-only, local cache. - // You can use DeepCopy() to make a deep copy of original object and modify this copy - // Or create a copy manually for better performance - imageCacheCopy := imageCache.DeepCopy() - imageCacheCopy.Spec = spec - imageCacheCopy.Status = *status - - if status.Status == fledgedv1alpha1.ImageCacheActionStatusFailed && - status.Reason == fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed { - imageCacheCopy.Annotations = make(map[string]string) - imageCacheCopy.Annotations[fledgedCacheSpecValidationKey] = "failed" - } - - if imageCacheCopy.Status.Status != fledgedv1alpha1.ImageCacheActionStatusProcessing { - completionTime := metav1.Now() - imageCacheCopy.Status.CompletionTime = &completionTime - } - // If the CustomResourceSubresources feature gate is not enabled, - // we must use Update instead of UpdateStatus to update the Status block of the ImageCache resource. - // UpdateStatus will not allow changes to the Spec of the resource, - // which is ideal for ensuring nothing other than resource status has been updated. - _, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(imageCache.Namespace).Update(imageCacheCopy) - return err -} - -func (c *Controller) removeAnnotation(imageCache *fledgedv1alpha1.ImageCache, annotationKey string) error { - imageCacheCopy := imageCache.DeepCopy() - delete(imageCacheCopy.Annotations, annotationKey) - _, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(imageCache.Namespace).Update(imageCacheCopy) - if err == nil { - glog.Infof("Annotation %s removed from imagecache(%s)", fledgedCacheSpecValidationKey, imageCache.Name) - } - return err -} +/* +Copyright 2018 The kube-fledged authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + "reflect" + "time" + + "github.com/golang/glog" + fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + clientset "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned" + fledgedscheme "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/scheme" + informers "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions/fledged/v1alpha1" + listers "github.com/senthilrch/kube-fledged/pkg/client/listers/fledged/v1alpha1" + "github.com/senthilrch/kube-fledged/pkg/images" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + coreinformers "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" +) + +const controllerAgentName = "fledged" +const fledgedCacheSpecValidationKey = "fledged.k8s.io/cachespecvalidation" +const imageCachePurgeAnnotationKey = "fledged.k8s.io/purge-imagecache" +const imageCacheRefreshAnnotationKey = "fledged.k8s.io/refresh-imagecache" + +const ( + // SuccessSynced is used as part of the Event 'reason' when a ImageCache is synced + SuccessSynced = "Synced" + // MessageResourceSynced is the message used for an Event fired when a ImageCache + // is synced successfully + MessageResourceSynced = "ImageCache synced successfully" +) + +// Controller is the controller for ImageCache resources +type Controller struct { + // kubeclientset is a standard kubernetes clientset + kubeclientset kubernetes.Interface + // fledgedclientset is a clientset for fledged.k8s.io API group + fledgedclientset clientset.Interface + + fledgedNameSpace string + nodesLister corelisters.NodeLister + nodesSynced cache.InformerSynced + imageCachesLister listers.ImageCacheLister + imageCachesSynced cache.InformerSynced + + // workqueue is a rate limited work queue. This is used to queue work to be + // processed instead of performing it as soon as a change happens. This + // means we can ensure we only process a fixed amount of resources at a + // time, and makes it easy to ensure we are never processing the same item + // simultaneously in two different workers. + workqueue workqueue.RateLimitingInterface + imageworkqueue workqueue.RateLimitingInterface + imageManager *images.ImageManager + // recorder is an event recorder for recording Event resources to the + // Kubernetes API. + recorder record.EventRecorder + imageCacheRefreshFrequency time.Duration +} + +// NewController returns a new fledged controller +func NewController( + kubeclientset kubernetes.Interface, + fledgedclientset clientset.Interface, + namespace string, + nodeInformer coreinformers.NodeInformer, + imageCacheInformer informers.ImageCacheInformer, + imageCacheRefreshFrequency time.Duration, + imagePullDeadlineDuration time.Duration, + dockerClientImage string, + imagePullPolicy string) *Controller { + + utilruntime.Must(fledgedscheme.AddToScheme(scheme.Scheme)) + glog.V(4).Info("Creating event broadcaster") + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: kubeclientset.CoreV1().Events("")}) + recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerAgentName}) + + controller := &Controller{ + kubeclientset: kubeclientset, + fledgedclientset: fledgedclientset, + fledgedNameSpace: namespace, + nodesLister: nodeInformer.Lister(), + nodesSynced: nodeInformer.Informer().HasSynced, + imageCachesLister: imageCacheInformer.Lister(), + imageCachesSynced: imageCacheInformer.Informer().HasSynced, + workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ImageCaches"), + imageworkqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ImagePullerStatus"), + recorder: recorder, + imageCacheRefreshFrequency: imageCacheRefreshFrequency, + } + + imageManager, _ := images.NewImageManager(controller.workqueue, controller.imageworkqueue, controller.kubeclientset, controller.fledgedNameSpace, imagePullDeadlineDuration, dockerClientImage, imagePullPolicy) + controller.imageManager = imageManager + + glog.Info("Setting up event handlers") + // Set up an event handler for when ImageCache resources change + imageCacheInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + controller.enqueueImageCache(images.ImageCacheCreate, nil, obj) + }, + UpdateFunc: func(old, new interface{}) { + controller.enqueueImageCache(images.ImageCacheUpdate, old, new) + }, + DeleteFunc: func(obj interface{}) { + controller.enqueueImageCache(images.ImageCacheDelete, obj, nil) + }, + }) + return controller +} + +// PreFlightChecks performs pre-flight checks and actions before the controller is started +func (c *Controller) PreFlightChecks() error { + if err := c.danglingJobs(); err != nil { + return err + } + if err := c.danglingImageCaches(); err != nil { + return err + } + return nil +} + +// danglingJobs finds and removes dangling or stuck jobs +func (c *Controller) danglingJobs() error { + joblist, err := c.kubeclientset.BatchV1().Jobs(c.fledgedNameSpace).List(metav1.ListOptions{}) + if err != nil { + glog.Errorf("Error listing jobs: %v", err) + return err + } + + if joblist == nil || len(joblist.Items) == 0 { + glog.Info("No dangling or stuck jobs found...") + return nil + } + deletePropagation := metav1.DeletePropagationBackground + for _, job := range joblist.Items { + err := c.kubeclientset.BatchV1().Jobs(c.fledgedNameSpace). + Delete(job.Name, &metav1.DeleteOptions{PropagationPolicy: &deletePropagation}) + if err != nil { + glog.Errorf("Error deleting job(%s): %v", job.Name, err) + return err + } + glog.Infof("Dangling Job(%s) deleted", job.Name) + } + return nil +} + +// danglingImageCaches finds dangling or stuck image cache and marks them as abhorted. Such +// image caches will get refreshed in the next cycle +func (c *Controller) danglingImageCaches() error { + dangling := false + imagecachelist, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(c.fledgedNameSpace).List(metav1.ListOptions{}) + if err != nil { + glog.Errorf("Error listing imagecaches: %v", err) + return err + } + + if imagecachelist == nil || len(imagecachelist.Items) == 0 { + glog.Info("No dangling or stuck imagecaches found...") + return nil + } + status := &fledgedv1alpha1.ImageCacheStatus{ + Failures: map[string]fledgedv1alpha1.NodeReasonMessageList{}, + Status: fledgedv1alpha1.ImageCacheActionStatusAborted, + Reason: fledgedv1alpha1.ImageCacheReasonImagePullAborted, + Message: fledgedv1alpha1.ImageCacheMessageImagePullAborted, + } + for _, imagecache := range imagecachelist.Items { + if imagecache.Status.Status == fledgedv1alpha1.ImageCacheActionStatusProcessing { + status.StartTime = imagecache.Status.StartTime + err := c.updateImageCacheStatus(&imagecache, status) + if err != nil { + glog.Errorf("Error updating ImageCache(%s) status to '%s': %v", imagecache.Name, fledgedv1alpha1.ImageCacheActionStatusAborted, err) + return err + } + dangling = true + glog.Infof("Dangling Image cache(%s) status changed to '%s'", imagecache.Name, fledgedv1alpha1.ImageCacheActionStatusAborted) + } + } + + if !dangling { + glog.Info("No dangling or stuck imagecaches found...") + } + return nil +} + +// Run will set up the event handlers for types we are interested in, as well +// as syncing informer caches and starting workers. It will block until stopCh +// is closed, at which point it will shutdown the workqueue and wait for +// workers to finish processing their current work items. +func (c *Controller) Run(threadiness int, stopCh <-chan struct{}) error { + defer runtime.HandleCrash() + defer c.workqueue.ShutDown() + defer c.imageworkqueue.ShutDown() + + // Start the informer factories to begin populating the informer caches + glog.Info("Starting fledged controller") + + // Wait for the caches to be synced before starting workers + glog.Info("Waiting for informer caches to sync") + if ok := cache.WaitForCacheSync(stopCh, c.nodesSynced, c.imageCachesSynced); !ok { + return fmt.Errorf("failed to wait for caches to sync") + } + + glog.Info("Starting image cache worker") + // Launch workers to process ImageCache resources + for i := 0; i < threadiness; i++ { + go wait.Until(c.runWorker, time.Second, stopCh) + } + + if c.imageCacheRefreshFrequency.Nanoseconds() != int64(0) { + glog.Info("Starting cache refresh worker") + go wait.Until(c.runRefreshWorker, c.imageCacheRefreshFrequency, stopCh) + } + + glog.Info("Started workers") + c.imageManager.Run(stopCh) + if err := c.imageManager.Run(stopCh); err != nil { + glog.Fatalf("Error running image manager: %s", err.Error()) + } + + <-stopCh + glog.Info("Shutting down workers") + + return nil +} + +// enqueueImageCache takes a ImageCache resource and converts it into a namespace/name +// string which is then put onto the work queue. This method should *not* be +// passed resources of any type other than ImageCache. +func (c *Controller) enqueueImageCache(workType images.WorkType, old, new interface{}) bool { + var key string + var err error + var obj interface{} + wqKey := images.WorkQueueKey{} + + switch workType { + case images.ImageCacheCreate: + obj = new + newImageCache := new.(*fledgedv1alpha1.ImageCache) + // If the ImageCache resource already has a status field, it means it's already + // synced, so do not queue it for processing + if !reflect.DeepEqual(newImageCache.Status, fledgedv1alpha1.ImageCacheStatus{}) { + return false + } + case images.ImageCacheUpdate: + obj = new + oldImageCache := old.(*fledgedv1alpha1.ImageCache) + newImageCache := new.(*fledgedv1alpha1.ImageCache) + + if oldImageCache.Status.Status == fledgedv1alpha1.ImageCacheActionStatusProcessing { + if !reflect.DeepEqual(newImageCache.Spec, oldImageCache.Spec) { + glog.Warningf("Received image cache update/purge/delete for '%s' while it is under processing, so ignoring.", oldImageCache.Name) + return false + } + } + if _, exists := newImageCache.Annotations[imageCachePurgeAnnotationKey]; exists { + if _, exists := oldImageCache.Annotations[imageCachePurgeAnnotationKey]; !exists { + workType = images.ImageCachePurge + break + } + } + if _, exists := newImageCache.Annotations[imageCacheRefreshAnnotationKey]; exists { + if _, exists := oldImageCache.Annotations[imageCacheRefreshAnnotationKey]; !exists { + workType = images.ImageCacheRefresh + break + } + } + if !reflect.DeepEqual(newImageCache.Spec, oldImageCache.Spec) { + if validation, ok := newImageCache.Annotations[fledgedCacheSpecValidationKey]; ok { + if validation == "failed" { + if err := c.removeAnnotation(newImageCache, fledgedCacheSpecValidationKey); err != nil { + glog.Errorf("Error removing Annotation %s from imagecache(%s): %v", fledgedCacheSpecValidationKey, newImageCache.Name, err) + } + return false + } + } + } else { + return false + } + case images.ImageCacheDelete: + return false + + case images.ImageCacheRefresh: + obj = old + } + + if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { + runtime.HandleError(err) + return false + } + wqKey.WorkType = workType + wqKey.ObjKey = key + if workType == images.ImageCacheUpdate { + oldImageCache := old.(*fledgedv1alpha1.ImageCache) + wqKey.OldImageCache = oldImageCache + } + + c.workqueue.AddRateLimited(wqKey) + glog.V(4).Infof("enqueueImageCache::ImageCache resource queued for work type %s", workType) + return true +} + +// runWorker is a long-running function that will continually call the +// processNextWorkItem function in order to read and process a message on the +// workqueue. +func (c *Controller) runWorker() { + for c.processNextWorkItem() { + } +} + +// processNextWorkItem will read a single work item off the workqueue and +// attempt to process it, by calling the syncHandler. +func (c *Controller) processNextWorkItem() bool { + //glog.Info("processNextWorkItem::Beginning...") + obj, shutdown := c.workqueue.Get() + + if shutdown { + return false + } + + // We wrap this block in a func so we can defer c.workqueue.Done. + err := func(obj interface{}) error { + // We call Done here so the workqueue knows we have finished + // processing this item. We also must remember to call Forget if we + // do not want this work item being re-queued. For example, we do + // not call Forget if a transient error occurs, instead the item is + // put back on the workqueue and attempted again after a back-off + // period. + defer c.workqueue.Done(obj) + var key images.WorkQueueKey + var ok bool + // We expect strings to come off the workqueue. These are of the + // form namespace/name. We do this as the delayed nature of the + // workqueue means the items in the informer cache may actually be + // more up to date that when the item was initially put onto the + // workqueue. + if key, ok = obj.(images.WorkQueueKey); !ok { + // As the item in the workqueue is actually invalid, we call + // Forget here else we'd go into a loop of attempting to + // process a work item that is invalid. + c.workqueue.Forget(obj) + runtime.HandleError(fmt.Errorf("Unexpected type in workqueue: %#v", obj)) + return nil + } + // Run the syncHandler, passing it the namespace/name string of the + // ImageCache resource to be synced. + if err := c.syncHandler(key); err != nil { + glog.Errorf("error syncing imagecache: %v", err.Error()) + return fmt.Errorf("error syncing imagecache: %v", err.Error()) + } + // Finally, if no error occurs we Forget this item so it does not + // get queued again until another change happens. + c.workqueue.Forget(obj) + //glog.Infof("Successfully synced '%s' for event '%s'", key.ObjKey, key.WorkType) + return nil + }(obj) + + if err != nil { + runtime.HandleError(err) + return true + } + + return true +} + +// runRefreshWorker is resposible of refreshing the image cache +func (c *Controller) runRefreshWorker() { + // List the ImageCache resources + imageCaches, err := c.imageCachesLister.ImageCaches(c.fledgedNameSpace).List(labels.Everything()) + if err != nil { + glog.Errorf("Error in listing image caches: %v", err) + return + } + for i := range imageCaches { + // Do not refresh if status is not yet updated + if reflect.DeepEqual(imageCaches[i].Status, fledgedv1alpha1.ImageCacheStatus{}) { + continue + } + // Do not refresh if image cache is already under processing + if imageCaches[i].Status.Status == fledgedv1alpha1.ImageCacheActionStatusProcessing { + continue + } + // Do not refresh image cache if cache spec validation failed + if imageCaches[i].Status.Status == fledgedv1alpha1.ImageCacheActionStatusFailed && + imageCaches[i].Status.Reason == fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed { + continue + } + // Do not refresh if image cache has been purged + if imageCaches[i].Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCachePurge { + continue + } + c.enqueueImageCache(images.ImageCacheRefresh, imageCaches[i], nil) + } +} + +// syncHandler compares the actual state with the desired, and attempts to +// converge the two. It then updates the Status block of the ImageCache resource +// with the current status of the resource. +func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error { + status := &fledgedv1alpha1.ImageCacheStatus{ + Failures: map[string]fledgedv1alpha1.NodeReasonMessageList{}, + } + + // Convert the namespace/name string into a distinct namespace and name + namespace, name, err := cache.SplitMetaNamespaceKey(wqKey.ObjKey) + if err != nil { + glog.Errorf("Error from cache.SplitMetaNamespaceKey(): %v", err) + return err + } + + glog.Infof("Starting to sync image cache %s(%s)", name, wqKey.WorkType) + + switch wqKey.WorkType { + case images.ImageCacheCreate, images.ImageCacheUpdate, images.ImageCacheRefresh, images.ImageCachePurge: + + startTime := metav1.Now() + status.StartTime = &startTime + // Get the ImageCache resource with this namespace/name + imageCache, err := c.imageCachesLister.ImageCaches(namespace).Get(name) + if err != nil { + // The ImageCache resource may no longer exist, in which case we stop + // processing. + glog.Errorf("Error getting imagecache(%s): %v", name, err) + return err + } + + err = validateCacheSpec(c, imageCache) + if err != nil { + status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed + status.Reason = fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed + status.Message = err.Error() + + if err := c.updateImageCacheStatus(imageCache, status); err != nil { + glog.Errorf("Error updating imagecache status to %s: %v", status.Status, err) + return err + } + + return err + } + + if wqKey.WorkType == images.ImageCacheUpdate && wqKey.OldImageCache == nil { + status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed + status.Reason = fledgedv1alpha1.ImageCacheReasonOldImageCacheNotFound + status.Message = fledgedv1alpha1.ImageCacheMessageOldImageCacheNotFound + + if err := c.updateImageCacheStatus(imageCache, status); err != nil { + glog.Errorf("Error updating imagecache status to %s: %v", status.Status, err) + return err + } + glog.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonOldImageCacheNotFound, fledgedv1alpha1.ImageCacheMessageOldImageCacheNotFound) + return fmt.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonOldImageCacheNotFound, fledgedv1alpha1.ImageCacheMessageOldImageCacheNotFound) + } + + if wqKey.WorkType == images.ImageCacheUpdate { + if len(wqKey.OldImageCache.Spec.CacheSpec) != len(imageCache.Spec.CacheSpec) { + status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed + status.Reason = fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed + status.Message = fledgedv1alpha1.ImageCacheMessageNotSupportedUpdates + + if err = c.updateImageCacheSpecAndStatus(imageCache, wqKey.OldImageCache.Spec, status); err != nil { + glog.Errorf("Error updating imagecache spec and status to %s: %v", status.Status, err) + return err + } + glog.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, "Mismatch in no. of image lists") + return fmt.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, "Mismatch in no. of image lists") + } + + for i := range wqKey.OldImageCache.Spec.CacheSpec { + if !reflect.DeepEqual(wqKey.OldImageCache.Spec.CacheSpec[i].NodeSelector, imageCache.Spec.CacheSpec[i].NodeSelector) { + status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed + status.Reason = fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed + status.Message = fledgedv1alpha1.ImageCacheMessageNotSupportedUpdates + + if err = c.updateImageCacheSpecAndStatus(imageCache, wqKey.OldImageCache.Spec, status); err != nil { + glog.Errorf("Error updating imagecache spec and status to %s: %v", status.Status, err) + return err + } + glog.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, "Mismatch in node selector") + return fmt.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, "Mismatch in node selector") + } + } + } + + cacheSpec := imageCache.Spec.CacheSpec + glog.V(4).Infof("cacheSpec: %+v", cacheSpec) + var nodes []*corev1.Node + + status.Status = fledgedv1alpha1.ImageCacheActionStatusProcessing + + if wqKey.WorkType == images.ImageCacheCreate { + status.Reason = fledgedv1alpha1.ImageCacheReasonImageCacheCreate + status.Message = fledgedv1alpha1.ImageCacheMessagePullingImages + } + + if wqKey.WorkType == images.ImageCacheUpdate { + status.Reason = fledgedv1alpha1.ImageCacheReasonImageCacheUpdate + status.Message = fledgedv1alpha1.ImageCacheMessageUpdatingCache + } + + if wqKey.WorkType == images.ImageCacheRefresh { + status.Reason = fledgedv1alpha1.ImageCacheReasonImageCacheRefresh + status.Message = fledgedv1alpha1.ImageCacheMessageRefreshingCache + } + + if wqKey.WorkType == images.ImageCachePurge { + status.Reason = fledgedv1alpha1.ImageCacheReasonImageCachePurge + status.Message = fledgedv1alpha1.ImageCacheMessagePurgeCache + } + + imageCache, err = c.fledgedclientset.FledgedV1alpha1().ImageCaches(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + glog.Errorf("Error getting imagecache(%s) from api server: %v", name, err) + return err + } + + if err = c.updateImageCacheStatus(imageCache, status); err != nil { + glog.Errorf("Error updating imagecache status to %s: %v", status.Status, err) + return err + } + + for k, i := range cacheSpec { + if len(i.NodeSelector) > 0 { + if nodes, err = c.nodesLister.List(labels.Set(i.NodeSelector).AsSelector()); err != nil { + glog.Errorf("Error listing nodes using nodeselector %+v: %v", i.NodeSelector, err) + return err + } + } else { + if nodes, err = c.nodesLister.List(labels.Everything()); err != nil { + glog.Errorf("Error listing nodes using nodeselector labels.Everything(): %v", err) + return err + } + } + glog.V(4).Infof("No. of nodes in %+v is %d", i.NodeSelector, len(nodes)) + if len(nodes) == 0 { + glog.Errorf("NodeSelector %+v did not match any nodes.", i.NodeSelector) + return fmt.Errorf("NodeSelector %+v did not match any nodes", i.NodeSelector) + } + + for _, n := range nodes { + for m := range i.Images { + ipr := images.ImageWorkRequest{ + Image: i.Images[m], + Node: n, + ContainerRuntimeVersion: n.Status.NodeInfo.ContainerRuntimeVersion, + WorkType: wqKey.WorkType, + Imagecache: imageCache, + } + c.imageworkqueue.AddRateLimited(ipr) + } + if wqKey.WorkType == images.ImageCacheUpdate { + for _, oldimage := range wqKey.OldImageCache.Spec.CacheSpec[k].Images { + matched := false + for _, newimage := range i.Images { + if oldimage == newimage { + matched = true + break + } + } + if !matched { + ipr := images.ImageWorkRequest{ + Image: oldimage, + Node: n, + ContainerRuntimeVersion: n.Status.NodeInfo.ContainerRuntimeVersion, + WorkType: images.ImageCachePurge, + Imagecache: imageCache, + } + c.imageworkqueue.AddRateLimited(ipr) + } + } + } + } + } + + // We add an empty image pull request to signal the image manager that all + // requests for this sync action have been placed in the imageworkqueue + c.imageworkqueue.AddRateLimited(images.ImageWorkRequest{WorkType: wqKey.WorkType, Imagecache: imageCache}) + + case images.ImageCacheStatusUpdate: + glog.V(4).Infof("wqKey.Status = %+v", wqKey.Status) + // Finally, we update the status block of the ImageCache resource to reflect the + // current state of the world + // Get the ImageCache resource with this namespace/name + imageCache, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + glog.Errorf("Error getting image cache %s: %v", name, err) + return err + } + + if imageCache.Status.StartTime != nil { + status.StartTime = imageCache.Status.StartTime + } + + status.Reason = imageCache.Status.Reason + + failures := false + for _, v := range *wqKey.Status { + if (v.Status == images.ImageWorkResultStatusSucceeded || v.Status == images.ImageWorkResultStatusAlreadyPulled) && !failures { + status.Status = fledgedv1alpha1.ImageCacheActionStatusSucceeded + if v.ImageWorkRequest.WorkType == images.ImageCachePurge { + status.Message = fledgedv1alpha1.ImageCacheMessageImagesDeletedSuccessfully + } else { + status.Message = fledgedv1alpha1.ImageCacheMessageImagesPulledSuccessfully + } + } + if v.Status == images.ImageWorkResultStatusFailed && !failures { + failures = true + status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed + if v.ImageWorkRequest.WorkType == images.ImageCachePurge { + status.Message = fledgedv1alpha1.ImageCacheMessageImageDeleteFailedForSomeImages + } else { + status.Message = fledgedv1alpha1.ImageCacheMessageImagePullFailedForSomeImages + } + } + if v.Status == images.ImageWorkResultStatusFailed { + status.Failures[v.ImageWorkRequest.Image] = append( + status.Failures[v.ImageWorkRequest.Image], fledgedv1alpha1.NodeReasonMessage{ + Node: v.ImageWorkRequest.Node.Labels["kubernetes.io/hostname"], + Reason: v.Reason, + Message: v.Message, + }) + } + } + + err = c.updateImageCacheStatus(imageCache, status) + if err != nil { + glog.Errorf("Error updating ImageCache status: %v", err) + return err + } + + if imageCache.Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCachePurge || imageCache.Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCacheRefresh { + imageCache, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + glog.Errorf("Error getting image cache %s: %v", name, err) + return err + } + if imageCache.Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCachePurge { + if err := c.removeAnnotation(imageCache, imageCachePurgeAnnotationKey); err != nil { + glog.Errorf("Error removing Annotation %s from imagecache(%s): %v", imageCachePurgeAnnotationKey, imageCache.Name, err) + return err + } + } + if imageCache.Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCacheRefresh { + if err := c.removeAnnotation(imageCache, imageCacheRefreshAnnotationKey); err != nil { + glog.Errorf("Error removing Annotation %s from imagecache(%s): %v", imageCacheRefreshAnnotationKey, imageCache.Name, err) + return err + } + } + } + + if status.Status == fledgedv1alpha1.ImageCacheActionStatusSucceeded { + c.recorder.Event(imageCache, corev1.EventTypeNormal, status.Reason, status.Message) + } + + if status.Status == fledgedv1alpha1.ImageCacheActionStatusFailed { + c.recorder.Event(imageCache, corev1.EventTypeWarning, status.Reason, status.Message) + } + } + glog.Infof("Completed sync actions for image cache %s(%s)", name, wqKey.WorkType) + return nil + +} + +func (c *Controller) updateImageCacheStatus(imageCache *fledgedv1alpha1.ImageCache, status *fledgedv1alpha1.ImageCacheStatus) error { + // NEVER modify objects from the store. It's a read-only, local cache. + // You can use DeepCopy() to make a deep copy of original object and modify this copy + // Or create a copy manually for better performance + imageCacheCopy := imageCache.DeepCopy() + imageCacheCopy.Status = *status + if imageCacheCopy.Status.Status != fledgedv1alpha1.ImageCacheActionStatusProcessing { + completionTime := metav1.Now() + imageCacheCopy.Status.CompletionTime = &completionTime + } + // If the CustomResourceSubresources feature gate is not enabled, + // we must use Update instead of UpdateStatus to update the Status block of the ImageCache resource. + // UpdateStatus will not allow changes to the Spec of the resource, + // which is ideal for ensuring nothing other than resource status has been updated. + _, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(imageCache.Namespace).Update(imageCacheCopy) + return err +} + +func (c *Controller) updateImageCacheSpecAndStatus(imageCache *fledgedv1alpha1.ImageCache, spec fledgedv1alpha1.ImageCacheSpec, status *fledgedv1alpha1.ImageCacheStatus) error { + // NEVER modify objects from the store. It's a read-only, local cache. + // You can use DeepCopy() to make a deep copy of original object and modify this copy + // Or create a copy manually for better performance + imageCacheCopy := imageCache.DeepCopy() + imageCacheCopy.Spec = spec + imageCacheCopy.Status = *status + + if status.Status == fledgedv1alpha1.ImageCacheActionStatusFailed && + status.Reason == fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed { + imageCacheCopy.Annotations = make(map[string]string) + imageCacheCopy.Annotations[fledgedCacheSpecValidationKey] = "failed" + } + + if imageCacheCopy.Status.Status != fledgedv1alpha1.ImageCacheActionStatusProcessing { + completionTime := metav1.Now() + imageCacheCopy.Status.CompletionTime = &completionTime + } + // If the CustomResourceSubresources feature gate is not enabled, + // we must use Update instead of UpdateStatus to update the Status block of the ImageCache resource. + // UpdateStatus will not allow changes to the Spec of the resource, + // which is ideal for ensuring nothing other than resource status has been updated. + _, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(imageCache.Namespace).Update(imageCacheCopy) + return err +} + +func (c *Controller) removeAnnotation(imageCache *fledgedv1alpha1.ImageCache, annotationKey string) error { + imageCacheCopy := imageCache.DeepCopy() + delete(imageCacheCopy.Annotations, annotationKey) + _, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(imageCache.Namespace).Update(imageCacheCopy) + if err == nil { + glog.Infof("Annotation %s removed from imagecache(%s)", fledgedCacheSpecValidationKey, imageCache.Name) + } + return err +} diff --git a/cmd/app/controller_helpers.go b/cmd/fledged/app/controller_helpers.go similarity index 96% rename from cmd/app/controller_helpers.go rename to cmd/fledged/app/controller_helpers.go index dbe08fbf..c8fa5ea0 100644 --- a/cmd/app/controller_helpers.go +++ b/cmd/fledged/app/controller_helpers.go @@ -1,73 +1,73 @@ -/* -Copyright 2018 The kube-fledged authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "fmt" - - "github.com/golang/glog" - fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" -) - -func validateCacheSpec(c *Controller, imageCache *fledgedv1alpha1.ImageCache) error { - if imageCache == nil { - glog.Errorf("Unable to obtain reference to image cache") - return fmt.Errorf("Unable to obtain reference to image cache") - } - - cacheSpec := imageCache.Spec.CacheSpec - glog.V(4).Infof("cacheSpec: %+v", cacheSpec) - var nodes []*corev1.Node - var err error - - for _, i := range cacheSpec { - if len(i.Images) == 0 { - glog.Error("No images specified within image list") - return fmt.Errorf("No images specified within image list") - - } - - for m := range i.Images { - for p := 0; p < m; p++ { - if i.Images[p] == i.Images[m] { - glog.Errorf("Duplicate image names within image list: %s", i.Images[m]) - return fmt.Errorf("Duplicate image names within image list: %s", i.Images[m]) - } - } - } - - if len(i.NodeSelector) > 0 { - if nodes, err = c.nodesLister.List(labels.Set(i.NodeSelector).AsSelector()); err != nil { - glog.Errorf("Error listing nodes using nodeselector %+v: %v", i.NodeSelector, err) - return err - } - } else { - if nodes, err = c.nodesLister.List(labels.Everything()); err != nil { - glog.Errorf("Error listing nodes using nodeselector labels.Everything(): %v", err) - return err - } - } - glog.V(4).Infof("No. of nodes in %+v is %d", i.NodeSelector, len(nodes)) - if len(nodes) == 0 { - glog.Errorf("NodeSelector %s did not match any nodes.", labels.Set(i.NodeSelector).String()) - return fmt.Errorf("NodeSelector %s did not match any nodes", labels.Set(i.NodeSelector).String()) - } - } - return nil -} +/* +Copyright 2018 The kube-fledged authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + + "github.com/golang/glog" + fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" +) + +func validateCacheSpec(c *Controller, imageCache *fledgedv1alpha1.ImageCache) error { + if imageCache == nil { + glog.Errorf("Unable to obtain reference to image cache") + return fmt.Errorf("Unable to obtain reference to image cache") + } + + cacheSpec := imageCache.Spec.CacheSpec + glog.V(4).Infof("cacheSpec: %+v", cacheSpec) + var nodes []*corev1.Node + var err error + + for _, i := range cacheSpec { + if len(i.Images) == 0 { + glog.Error("No images specified within image list") + return fmt.Errorf("No images specified within image list") + + } + + for m := range i.Images { + for p := 0; p < m; p++ { + if i.Images[p] == i.Images[m] { + glog.Errorf("Duplicate image names within image list: %s", i.Images[m]) + return fmt.Errorf("Duplicate image names within image list: %s", i.Images[m]) + } + } + } + + if len(i.NodeSelector) > 0 { + if nodes, err = c.nodesLister.List(labels.Set(i.NodeSelector).AsSelector()); err != nil { + glog.Errorf("Error listing nodes using nodeselector %+v: %v", i.NodeSelector, err) + return err + } + } else { + if nodes, err = c.nodesLister.List(labels.Everything()); err != nil { + glog.Errorf("Error listing nodes using nodeselector labels.Everything(): %v", err) + return err + } + } + glog.V(4).Infof("No. of nodes in %+v is %d", i.NodeSelector, len(nodes)) + if len(nodes) == 0 { + glog.Errorf("NodeSelector %s did not match any nodes.", labels.Set(i.NodeSelector).String()) + return fmt.Errorf("NodeSelector %s did not match any nodes", labels.Set(i.NodeSelector).String()) + } + } + return nil +} diff --git a/cmd/app/controller_helpers_test.go b/cmd/fledged/app/controller_helpers_test.go similarity index 96% rename from cmd/app/controller_helpers_test.go rename to cmd/fledged/app/controller_helpers_test.go index 1c42c603..6c4d01f5 100644 --- a/cmd/app/controller_helpers_test.go +++ b/cmd/fledged/app/controller_helpers_test.go @@ -1,192 +1,192 @@ -/* -Copyright 2018 The kube-fledged authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "fmt" - "strings" - "testing" - - fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" - fledgedclientsetfake "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/fake" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - fakeclientset "k8s.io/client-go/kubernetes/fake" -) - -var ( - alwaysReady = func() bool { return true } -) - -func TestValidateCacheSpec(t *testing.T) { - //var fakekubeclientset *fakeclientset.Clientset - //var fakefledgedclientset *fledgedclientsetfake.Clientset - - tests := []struct { - name string - imageCache *fledgedv1alpha1.ImageCache - nodeList *corev1.NodeList - nodeListError error - expectErr bool - errorString string - }{ - { - name: "Unable to obtain reference to image cache", - imageCache: nil, - nodeList: nil, - nodeListError: nil, - expectErr: true, - errorString: "Unable to obtain reference to image cache", - }, - { - name: "No images specified within image list", - imageCache: &fledgedv1alpha1.ImageCache{ - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{}, - }, - }, - }, - }, - nodeList: nil, - nodeListError: nil, - expectErr: true, - errorString: "No images specified within image list", - }, - { - name: "Duplicate image names within image list", - imageCache: &fledgedv1alpha1.ImageCache{ - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo", "foo"}, - }, - }, - }, - }, - nodeList: nil, - nodeListError: nil, - expectErr: true, - errorString: "Duplicate image names within image list", - }, - { - name: "Error listing nodes using nodeselector", - imageCache: &fledgedv1alpha1.ImageCache{ - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - NodeSelector: map[string]string{"foo": "bar"}, - }, - }, - }, - }, - nodeList: nil, - nodeListError: fmt.Errorf("fake error"), - expectErr: true, - errorString: "Error listing nodes using nodeselector", - }, - { - name: "Error listing nodes using nodeselector labels.Everything()", - imageCache: &fledgedv1alpha1.ImageCache{ - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - }, - nodeList: nil, - nodeListError: fmt.Errorf("fake error"), - expectErr: true, - errorString: "Error listing nodes using nodeselector labels.Everything()", - }, - { - name: "NodeSelector did not match any nodes", - imageCache: &fledgedv1alpha1.ImageCache{ - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - NodeSelector: map[string]string{"foo": "bar"}, - }, - }, - }, - }, - nodeList: &corev1.NodeList{}, - nodeListError: nil, - expectErr: true, - errorString: "NodeSelector foo=bar did not match any nodes", - }, - { - name: "Successful validation", - imageCache: &fledgedv1alpha1.ImageCache{ - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - NodeSelector: map[string]string{"foo": "bar"}, - }, - }, - }, - }, - nodeList: &corev1.NodeList{ - Items: []corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "fakenode", - Labels: map[string]string{"foo": "bar"}, - }, - }, - }, - }, - nodeListError: nil, - expectErr: false, - errorString: "", - }, - } - - for _, test := range tests { - fakekubeclientset := &fakeclientset.Clientset{} - fakefledgedclientset := &fledgedclientsetfake.Clientset{} - - controller, nodeInformer, _ := newTestController(fakekubeclientset, fakefledgedclientset) - - if test.nodeListError != nil { - //TODO: How to return a fake error from node Lister? - continue - } - - if test.nodeList != nil && len(test.nodeList.Items) > 0 { - for _, node := range test.nodeList.Items { - nodeInformer.Informer().GetIndexer().Add(&node) - } - } - - err := validateCacheSpec(controller, test.imageCache) - if test.expectErr { - if err != nil && strings.HasPrefix(err.Error(), test.errorString) { - } else { - t.Errorf("Test: %s failed", test.name) - } - } else if err != nil { - t.Errorf("Test: %s failed. err received = %s", test.name, err.Error()) - } - } -} +/* +Copyright 2018 The kube-fledged authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + "strings" + "testing" + + fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + fledgedclientsetfake "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/fake" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + fakeclientset "k8s.io/client-go/kubernetes/fake" +) + +var ( + alwaysReady = func() bool { return true } +) + +func TestValidateCacheSpec(t *testing.T) { + //var fakekubeclientset *fakeclientset.Clientset + //var fakefledgedclientset *fledgedclientsetfake.Clientset + + tests := []struct { + name string + imageCache *fledgedv1alpha1.ImageCache + nodeList *corev1.NodeList + nodeListError error + expectErr bool + errorString string + }{ + { + name: "Unable to obtain reference to image cache", + imageCache: nil, + nodeList: nil, + nodeListError: nil, + expectErr: true, + errorString: "Unable to obtain reference to image cache", + }, + { + name: "No images specified within image list", + imageCache: &fledgedv1alpha1.ImageCache{ + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{}, + }, + }, + }, + }, + nodeList: nil, + nodeListError: nil, + expectErr: true, + errorString: "No images specified within image list", + }, + { + name: "Duplicate image names within image list", + imageCache: &fledgedv1alpha1.ImageCache{ + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo", "foo"}, + }, + }, + }, + }, + nodeList: nil, + nodeListError: nil, + expectErr: true, + errorString: "Duplicate image names within image list", + }, + { + name: "Error listing nodes using nodeselector", + imageCache: &fledgedv1alpha1.ImageCache{ + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + NodeSelector: map[string]string{"foo": "bar"}, + }, + }, + }, + }, + nodeList: nil, + nodeListError: fmt.Errorf("fake error"), + expectErr: true, + errorString: "Error listing nodes using nodeselector", + }, + { + name: "Error listing nodes using nodeselector labels.Everything()", + imageCache: &fledgedv1alpha1.ImageCache{ + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + }, + nodeList: nil, + nodeListError: fmt.Errorf("fake error"), + expectErr: true, + errorString: "Error listing nodes using nodeselector labels.Everything()", + }, + { + name: "NodeSelector did not match any nodes", + imageCache: &fledgedv1alpha1.ImageCache{ + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + NodeSelector: map[string]string{"foo": "bar"}, + }, + }, + }, + }, + nodeList: &corev1.NodeList{}, + nodeListError: nil, + expectErr: true, + errorString: "NodeSelector foo=bar did not match any nodes", + }, + { + name: "Successful validation", + imageCache: &fledgedv1alpha1.ImageCache{ + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + NodeSelector: map[string]string{"foo": "bar"}, + }, + }, + }, + }, + nodeList: &corev1.NodeList{ + Items: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakenode", + Labels: map[string]string{"foo": "bar"}, + }, + }, + }, + }, + nodeListError: nil, + expectErr: false, + errorString: "", + }, + } + + for _, test := range tests { + fakekubeclientset := &fakeclientset.Clientset{} + fakefledgedclientset := &fledgedclientsetfake.Clientset{} + + controller, nodeInformer, _ := newTestController(fakekubeclientset, fakefledgedclientset) + + if test.nodeListError != nil { + //TODO: How to return a fake error from node Lister? + continue + } + + if test.nodeList != nil && len(test.nodeList.Items) > 0 { + for _, node := range test.nodeList.Items { + nodeInformer.Informer().GetIndexer().Add(&node) + } + } + + err := validateCacheSpec(controller, test.imageCache) + if test.expectErr { + if err != nil && strings.HasPrefix(err.Error(), test.errorString) { + } else { + t.Errorf("Test: %s failed", test.name) + } + } else if err != nil { + t.Errorf("Test: %s failed. err received = %s", test.name, err.Error()) + } + } +} diff --git a/cmd/app/controller_test.go b/cmd/fledged/app/controller_test.go similarity index 96% rename from cmd/app/controller_test.go rename to cmd/fledged/app/controller_test.go index 8befbcf2..8ae27204 100644 --- a/cmd/app/controller_test.go +++ b/cmd/fledged/app/controller_test.go @@ -1,1175 +1,1175 @@ -/* -Copyright 2018 The kube-fledged authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "fmt" - "strings" - "testing" - "time" - - fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" - clientset "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned" - fledgedclientsetfake "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/fake" - informers "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions" - fledgedinformers "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions/fledged/v1alpha1" - "github.com/senthilrch/kube-fledged/pkg/images" - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - kubeinformers "k8s.io/client-go/informers" - coreinformers "k8s.io/client-go/informers/core/v1" - "k8s.io/client-go/kubernetes" - fakeclientset "k8s.io/client-go/kubernetes/fake" - core "k8s.io/client-go/testing" -) - -const fledgedNameSpace = "kube-fledged" - -var node = corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"kubernetes.io/hostname": "bar"}, - }, -} - -// noResyncPeriodFunc returns 0 for resyncPeriod in case resyncing is not needed. -func noResyncPeriodFunc() time.Duration { - return 0 -} - -func newTestController(kubeclientset kubernetes.Interface, fledgedclientset clientset.Interface) (*Controller, coreinformers.NodeInformer, fledgedinformers.ImageCacheInformer) { - kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeclientset, noResyncPeriodFunc()) - fledgedInformerFactory := informers.NewSharedInformerFactory(fledgedclientset, noResyncPeriodFunc()) - nodeInformer := kubeInformerFactory.Core().V1().Nodes() - imagecacheInformer := fledgedInformerFactory.Fledged().V1alpha1().ImageCaches() - imageCacheRefreshFrequency := time.Second * 0 - imagePullDeadlineDuration := time.Second * 5 - dockerClientImage := "senthilrch/fledged-docker-client:latest" - imagePullPolicy := "IfNotPresent" - - /* startInformers := true - if startInformers { - stopCh := make(chan struct{}) - defer close(stopCh) - kubeInformerFactory.Start(stopCh) - fledgedInformerFactory.Start(stopCh) - } */ - - controller := NewController(kubeclientset, fledgedclientset, fledgedNameSpace, nodeInformer, imagecacheInformer, - imageCacheRefreshFrequency, imagePullDeadlineDuration, dockerClientImage, imagePullPolicy) - controller.nodesSynced = func() bool { return true } - controller.imageCachesSynced = func() bool { return true } - return controller, nodeInformer, imagecacheInformer -} - -func TestPreFlightChecks(t *testing.T) { - tests := []struct { - name string - jobList *batchv1.JobList - jobListError error - jobDeleteError error - imageCacheList *fledgedv1alpha1.ImageCacheList - imageCacheListError error - imageCacheUpdateError error - expectErr bool - errorString string - }{ - { - name: "#1: No dangling jobs. No imagecaches", - jobList: &batchv1.JobList{Items: []batchv1.Job{}}, - jobListError: nil, - jobDeleteError: nil, - imageCacheList: &fledgedv1alpha1.ImageCacheList{Items: []fledgedv1alpha1.ImageCache{}}, - imageCacheListError: nil, - imageCacheUpdateError: nil, - expectErr: false, - errorString: "", - }, - { - name: "#2: No dangling jobs. No dangling imagecaches", - jobList: &batchv1.JobList{Items: []batchv1.Job{}}, - jobListError: nil, - jobDeleteError: nil, - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, - }, - }, - }, - }, - imageCacheListError: nil, - imageCacheUpdateError: nil, - expectErr: false, - errorString: "", - }, - { - name: "#3: One dangling job. One dangling image cache. Successful list and delete", - //imageCache: nil, - jobList: &batchv1.JobList{ - Items: []batchv1.Job{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - }, - }, - }, - }, - jobListError: nil, - jobDeleteError: nil, - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, - }, - }, - }, - }, - imageCacheListError: nil, - imageCacheUpdateError: nil, - expectErr: false, - errorString: "", - }, - { - name: "#4: Unsuccessful listing of jobs", - jobList: nil, - jobListError: fmt.Errorf("fake error"), - jobDeleteError: nil, - expectErr: true, - errorString: "Internal error occurred: fake error", - }, - { - name: "#5: Unsuccessful listing of imagecaches", - jobList: &batchv1.JobList{Items: []batchv1.Job{}}, - jobListError: nil, - jobDeleteError: nil, - imageCacheList: nil, - imageCacheListError: fmt.Errorf("fake error"), - imageCacheUpdateError: nil, - expectErr: true, - errorString: "Internal error occurred: fake error", - }, - { - name: "#6: One dangling job. Successful list. Unsuccessful delete", - jobList: &batchv1.JobList{ - Items: []batchv1.Job{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - }, - }, - }, - }, - jobListError: nil, - jobDeleteError: fmt.Errorf("fake error"), - expectErr: true, - errorString: "Internal error occurred: fake error", - }, - { - name: "#7: One dangling image cache. Successful list. Unsuccessful delete", - jobList: &batchv1.JobList{Items: []batchv1.Job{}}, - jobListError: nil, - jobDeleteError: nil, - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, - }, - }, - }, - }, - imageCacheListError: nil, - imageCacheUpdateError: fmt.Errorf("fake error"), - expectErr: true, - errorString: "Internal error occurred: fake error", - }, - } - for _, test := range tests { - fakekubeclientset := &fakeclientset.Clientset{} - fakefledgedclientset := &fledgedclientsetfake.Clientset{} - if test.jobListError != nil { - listError := apierrors.NewInternalError(test.jobListError) - fakekubeclientset.AddReactor("list", "jobs", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, listError - }) - } else { - fakekubeclientset.AddReactor("list", "jobs", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, test.jobList, nil - }) - } - if test.jobDeleteError != nil { - deleteError := apierrors.NewInternalError(test.jobDeleteError) - fakekubeclientset.AddReactor("delete", "jobs", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, deleteError - }) - } else { - fakekubeclientset.AddReactor("delete", "jobs", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, nil - }) - } - - if test.imageCacheListError != nil { - listError := apierrors.NewInternalError(test.imageCacheListError) - fakefledgedclientset.AddReactor("list", "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, listError - }) - } else { - fakefledgedclientset.AddReactor("list", "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, test.imageCacheList, nil - }) - } - if test.imageCacheUpdateError != nil { - updateError := apierrors.NewInternalError(test.imageCacheUpdateError) - fakefledgedclientset.AddReactor("update", "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, updateError - }) - } else { - fakefledgedclientset.AddReactor("update", "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, nil - }) - } - - controller, _, _ := newTestController(fakekubeclientset, fakefledgedclientset) - - err := controller.PreFlightChecks() - if test.expectErr { - if !(err != nil && strings.HasPrefix(err.Error(), test.errorString)) { - t.Errorf("Test: %s failed", test.name) - } - } else { - if err != nil { - t.Errorf("Test: %s failed. err received = %s", test.name, err.Error()) - } - } - } - t.Logf("%d tests passed", len(tests)) -} - -func TestRunRefreshWorker(t *testing.T) { - tests := []struct { - name string - imageCacheList *fledgedv1alpha1.ImageCacheList - imageCacheListError error - workqueueItems int - }{ - { - name: "#1: Do not refresh if status is not yet updated", - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - }, - }, - }, - imageCacheListError: nil, - workqueueItems: 0, - }, - { - name: "#2: Do not refresh if image cache is already under processing", - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, - }, - }, - }, - }, - imageCacheListError: nil, - workqueueItems: 0, - }, - { - name: "#3: Do not refresh image cache if cache spec validation failed", - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusFailed, - Reason: fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, - }, - }, - }, - }, - imageCacheListError: nil, - workqueueItems: 0, - }, - { - name: "#4: Do not refresh if image cache has been purged", - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Reason: fledgedv1alpha1.ImageCacheReasonImageCachePurge, - }, - }, - }, - }, - imageCacheListError: nil, - workqueueItems: 0, - }, - { - name: "#5: Successfully queued 1 imagecache for refresh", - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, - }, - }, - }, - }, - imageCacheListError: nil, - workqueueItems: 1, - }, - { - name: "#6: Successfully queued 2 imagecaches for refresh", - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "bar", - Namespace: "kube-fledged", - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusFailed, - }, - }, - }, - }, - imageCacheListError: nil, - workqueueItems: 2, - }, - { - name: "#7: No imagecaches to refresh", - imageCacheList: nil, - imageCacheListError: nil, - workqueueItems: 0, - }, - } - - for _, test := range tests { - if test.workqueueItems > 0 { - //TODO: How to check if workqueue contains the added item? - continue - } - fakekubeclientset := &fakeclientset.Clientset{} - fakefledgedclientset := &fledgedclientsetfake.Clientset{} - - controller, _, imagecacheInformer := newTestController(fakekubeclientset, fakefledgedclientset) - if test.imageCacheList != nil && len(test.imageCacheList.Items) > 0 { - for _, imagecache := range test.imageCacheList.Items { - imagecacheInformer.Informer().GetIndexer().Add(&imagecache) - } - } - controller.runRefreshWorker() - if test.workqueueItems == controller.workqueue.Len() { - } else { - t.Errorf("Test: %s failed: expected %d, actual %d", test.name, test.workqueueItems, controller.workqueue.Len()) - } - } -} - -func TestSyncHandler(t *testing.T) { - type ActionReaction struct { - action string - reaction string - } - now := metav1.Now() - defaultImageCache := fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - } - defaultNodeList := &corev1.NodeList{ - Items: []corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "fakenode", - Labels: map[string]string{"kubernetes.io/hostname": "bar"}, - }, - }, - }, - } - - tests := []struct { - name string - imageCache fledgedv1alpha1.ImageCache - wqKey images.WorkQueueKey - nodeList *corev1.NodeList - expectedActions []ActionReaction - expectErr bool - expectedErrString string - }{ - { - name: "#1: Invalid imagecache resource key", - wqKey: images.WorkQueueKey{ - ObjKey: "foo/bar/car", - }, - expectErr: true, - expectedErrString: "unexpected key format", - }, - { - name: "#2: Create - Invalid imagecache spec (no images specified)", - imageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{}, - }, - }, - }, - }, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheCreate, - }, - expectedActions: []ActionReaction{{action: "update", reaction: ""}}, - expectErr: true, - expectedErrString: "No images specified within image list", - }, - { - name: "#3: Update - Old imagecache pointer is nil", - imageCache: defaultImageCache, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheUpdate, - OldImageCache: nil, - }, - nodeList: defaultNodeList, - expectedActions: []ActionReaction{{action: "update", reaction: ""}}, - expectErr: true, - expectedErrString: "OldImageCacheNotFound", - }, - { - name: "#4: Update - No. of imagelists not equal", - imageCache: defaultImageCache, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheUpdate, - OldImageCache: &fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - { - Images: []string{"bar"}, - }, - }, - }, - }, - }, - nodeList: defaultNodeList, - expectedActions: []ActionReaction{{action: "update", reaction: ""}}, - expectErr: true, - expectedErrString: "CacheSpecValidationFailed", - }, - { - name: "#5: Update - Change in NodeSelectors", - imageCache: defaultImageCache, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheUpdate, - OldImageCache: &fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - NodeSelector: map[string]string{"foo": "bar"}, - }, - }, - }, - }, - }, - nodeList: defaultNodeList, - expectedActions: []ActionReaction{{action: "update", reaction: ""}}, - expectErr: true, - expectedErrString: "CacheSpecValidationFailed", - }, - { - name: "#6: Refresh - Update status to processing", - imageCache: defaultImageCache, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheRefresh, - }, - nodeList: defaultNodeList, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: "fake error"}, - }, - expectErr: true, - expectedErrString: "Internal error occurred: fake error", - }, - { - name: "#7: StatusUpdate - Successful Refresh", - imageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - StartTime: &now, - Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, - Reason: fledgedv1alpha1.ImageCacheReasonImageCacheRefresh, - }, - }, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheStatusUpdate, - Status: &map[string]images.ImageWorkResult{ - "job1": { - Status: images.ImageWorkResultStatusSucceeded, - ImageWorkRequest: images.ImageWorkRequest{ - WorkType: images.ImageCacheRefresh, - Node: &node, - }, - }, - }, - }, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: ""}, - }, - expectErr: false, - expectedErrString: "", - }, - { - name: "#8: Purge - Update status to processing", - imageCache: defaultImageCache, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCachePurge, - }, - nodeList: defaultNodeList, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: "fake error"}, - }, - expectErr: true, - expectedErrString: "Internal error occurred: fake error", - }, - { - name: "#9: Purge - Successful purge", - imageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Reason: fledgedv1alpha1.ImageCacheReasonImageCachePurge, - }, - }, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheRefresh, - }, - nodeList: defaultNodeList, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: ""}, - }, - expectErr: false, - expectedErrString: "", - }, - { - name: "#10: Create - Successfully firing imagepull requests", - imageCache: defaultImageCache, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheCreate, - }, - nodeList: defaultNodeList, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: ""}, - }, - expectErr: false, - expectedErrString: "", - }, - { - name: "#11: Update - Successfully firing imagepull & imagedelete requests", - imageCache: defaultImageCache, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheUpdate, - OldImageCache: &fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo", "bar"}, - }, - }, - }, - }, - }, - nodeList: defaultNodeList, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: ""}, - }, - expectErr: false, - expectedErrString: "", - }, - { - name: "#12: StatusUpdate - ImagesPulledSuccessfully", - imageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - StartTime: &now, - }, - }, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheStatusUpdate, - Status: &map[string]images.ImageWorkResult{ - "job1": { - Status: images.ImageWorkResultStatusSucceeded, - ImageWorkRequest: images.ImageWorkRequest{ - WorkType: images.ImageCacheCreate, - Node: &node, - }, - }, - }, - }, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: ""}, - }, - expectErr: false, - expectedErrString: "", - }, - { - name: "#13: StatusUpdate - ImagesDeletedSuccessfully", - imageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - StartTime: &now, - Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, - Reason: fledgedv1alpha1.ImageCacheReasonImageCachePurge, - }, - }, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheStatusUpdate, - Status: &map[string]images.ImageWorkResult{ - "job1": { - Status: images.ImageWorkResultStatusSucceeded, - ImageWorkRequest: images.ImageWorkRequest{ - WorkType: images.ImageCachePurge, - Node: &node, - }, - }, - }, - }, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: ""}, - }, - expectErr: false, - expectedErrString: "", - }, - { - name: "#14: StatusUpdate - ImagePullFailedForSomeImages", - imageCache: defaultImageCache, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheStatusUpdate, - Status: &map[string]images.ImageWorkResult{ - "job1": { - Status: images.ImageWorkResultStatusFailed, - ImageWorkRequest: images.ImageWorkRequest{ - WorkType: images.ImageCacheCreate, - Node: &node, - }, - }, - }, - }, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: ""}, - }, - expectErr: false, - expectedErrString: "", - }, - { - name: "#15: StatusUpdate - ImageDeleteFailedForSomeImages", - imageCache: defaultImageCache, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheStatusUpdate, - Status: &map[string]images.ImageWorkResult{ - "job1": { - Status: images.ImageWorkResultStatusFailed, - ImageWorkRequest: images.ImageWorkRequest{ - WorkType: images.ImageCachePurge, - Node: &node, - }, - }, - }, - }, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: ""}, - }, - expectErr: false, - expectedErrString: "", - }, - } - - for _, test := range tests { - fakekubeclientset := &fakeclientset.Clientset{} - fakefledgedclientset := &fledgedclientsetfake.Clientset{} - for _, ar := range test.expectedActions { - if ar.reaction != "" { - apiError := apierrors.NewInternalError(fmt.Errorf(ar.reaction)) - fakefledgedclientset.AddReactor(ar.action, "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, apiError - }) - } - fakefledgedclientset.AddReactor(ar.action, "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, &test.imageCache, nil - }) - } - - controller, nodeInformer, imagecacheInformer := newTestController(fakekubeclientset, fakefledgedclientset) - if test.nodeList != nil && len(test.nodeList.Items) > 0 { - for _, node := range test.nodeList.Items { - nodeInformer.Informer().GetIndexer().Add(&node) - } - } - imagecacheInformer.Informer().GetIndexer().Add(&test.imageCache) - err := controller.syncHandler(test.wqKey) - if test.expectErr { - if err == nil { - t.Errorf("Test: %s failed: expectedError=%s, actualError=nil", test.name, test.expectedErrString) - } - if err != nil && !strings.HasPrefix(err.Error(), test.expectedErrString) { - t.Errorf("Test: %s failed: expectedError=%s, actualError=%s", test.name, test.expectedErrString, err.Error()) - } - } else if err != nil { - t.Errorf("Test: %s failed. expectedError=nil, actualError=%s", test.name, err.Error()) - } - } - t.Logf("%d tests passed", len(tests)) -} - -func TestEnqueueImageCache(t *testing.T) { - //now := metav1.Now() - //nowplus5s := metav1.NewTime(time.Now().Add(time.Second * 5)) - defaultImageCache := fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - } - tests := []struct { - name string - workType images.WorkType - oldImageCache fledgedv1alpha1.ImageCache - newImageCache fledgedv1alpha1.ImageCache - expectedResult bool - }{ - { - name: "#1: Create - Imagecache queued successfully", - workType: images.ImageCacheCreate, - newImageCache: defaultImageCache, - expectedResult: true, - }, - { - name: "#2: Create - Imagecache with Status field, so no queueing", - workType: images.ImageCacheCreate, - newImageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, - }, - }, - expectedResult: false, - }, - { - name: "#3: Update - Imagecache purge. Successful queueing", - workType: images.ImageCacheUpdate, - oldImageCache: defaultImageCache, - newImageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - Annotations: map[string]string{imageCachePurgeAnnotationKey: ""}, - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, - }, - }, - expectedResult: true, - }, - { - name: "#4: Update - No change in Spec. Unsuccessful queueing", - workType: images.ImageCacheUpdate, - oldImageCache: defaultImageCache, - newImageCache: defaultImageCache, - expectedResult: false, - }, - { - name: "#5: Update - Status processing. Unsuccessful queueing", - workType: images.ImageCacheUpdate, - oldImageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, - }, - }, - expectedResult: false, - }, - { - name: "#6: Update - Successful queueing", - workType: images.ImageCacheUpdate, - oldImageCache: defaultImageCache, - newImageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo", "bar"}, - }, - }, - }, - }, - expectedResult: true, - }, - { - name: "#7: Delete - Unsuccessful queueing", - workType: images.ImageCacheDelete, - expectedResult: false, - }, - { - name: "#8: Refresh - Successful queueing", - workType: images.ImageCacheRefresh, - oldImageCache: defaultImageCache, - expectedResult: true, - }, - { - name: "#9: Update - CacheSpec restoration", - workType: images.ImageCacheUpdate, - oldImageCache: defaultImageCache, - newImageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - Annotations: map[string]string{ - fledgedCacheSpecValidationKey: "failed", - }, - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo", "bar"}, - }, - }, - }, - }, - expectedResult: false, - }, - { - name: "#10: Update - Imagecache refresh. Successful queueing", - workType: images.ImageCacheUpdate, - oldImageCache: defaultImageCache, - newImageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - Annotations: map[string]string{imageCacheRefreshAnnotationKey: ""}, - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, - }, - }, - expectedResult: true, - }, - } - - for _, test := range tests { - fakekubeclientset := &fakeclientset.Clientset{} - fakefledgedclientset := &fledgedclientsetfake.Clientset{} - controller, _, _ := newTestController(fakekubeclientset, fakefledgedclientset) - result := controller.enqueueImageCache(test.workType, &test.oldImageCache, &test.newImageCache) - if result != test.expectedResult { - t.Errorf("Test %s failed: expected=%t, actual=%t", test.name, test.expectedResult, result) - } - } -} - -func TestProcessNextWorkItem(t *testing.T) { - type ActionReaction struct { - action string - reaction string - } - defaultImageCache := fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - } - - tests := []struct { - name string - imageCache fledgedv1alpha1.ImageCache - wqKey images.WorkQueueKey - expectedActions []ActionReaction - expectErr bool - expectedErrString string - }{ - { - name: "#1: StatusUpdate - ImageDeleteFailedForSomeImages", - imageCache: defaultImageCache, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheStatusUpdate, - Status: &map[string]images.ImageWorkResult{ - "job1": { - Status: images.ImageWorkResultStatusFailed, - ImageWorkRequest: images.ImageWorkRequest{ - WorkType: images.ImageCachePurge, - Node: &node, - }, - }, - }, - }, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: ""}, - }, - expectErr: false, - expectedErrString: "", - }, - { - name: "#2: Create - Invalid imagecache spec (no images specified)", - imageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{}, - }, - }, - }, - }, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheCreate, - }, - expectedActions: []ActionReaction{{action: "update", reaction: ""}}, - expectErr: false, - expectedErrString: "No images specified within image list", - }, - { - name: "#3: Unexpected type in workqueue", - expectErr: false, - expectedErrString: "Unexpected type in workqueue", - }, - } - - for _, test := range tests { - fakekubeclientset := &fakeclientset.Clientset{} - fakefledgedclientset := &fledgedclientsetfake.Clientset{} - for _, ar := range test.expectedActions { - if ar.reaction != "" { - apiError := apierrors.NewInternalError(fmt.Errorf(ar.reaction)) - fakefledgedclientset.AddReactor(ar.action, "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, apiError - }) - } - fakefledgedclientset.AddReactor(ar.action, "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, &test.imageCache, nil - }) - } - - controller, _, imagecacheInformer := newTestController(fakekubeclientset, fakefledgedclientset) - imagecacheInformer.Informer().GetIndexer().Add(&test.imageCache) - if test.expectedErrString == "Unexpected type in workqueue" { - controller.workqueue.Add(struct{}{}) - } - controller.workqueue.Add(test.wqKey) - controller.processNextWorkItem() - var err error - if test.expectErr { - if err == nil { - t.Errorf("Test: %s failed: expectedError=%s, actualError=nil", test.name, test.expectedErrString) - } - if err != nil && !strings.HasPrefix(err.Error(), test.expectedErrString) { - t.Errorf("Test: %s failed: expectedError=%s, actualError=%s", test.name, test.expectedErrString, err.Error()) - } - } else if err != nil { - t.Errorf("Test: %s failed. expectedError=nil, actualError=%s", test.name, err.Error()) - } - } - t.Logf("%d tests passed", len(tests)) -} +/* +Copyright 2018 The kube-fledged authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + "strings" + "testing" + "time" + + fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + clientset "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned" + fledgedclientsetfake "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/fake" + informers "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions" + fledgedinformers "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions/fledged/v1alpha1" + "github.com/senthilrch/kube-fledged/pkg/images" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubeinformers "k8s.io/client-go/informers" + coreinformers "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/kubernetes" + fakeclientset "k8s.io/client-go/kubernetes/fake" + core "k8s.io/client-go/testing" +) + +const fledgedNameSpace = "kube-fledged" + +var node = corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"kubernetes.io/hostname": "bar"}, + }, +} + +// noResyncPeriodFunc returns 0 for resyncPeriod in case resyncing is not needed. +func noResyncPeriodFunc() time.Duration { + return 0 +} + +func newTestController(kubeclientset kubernetes.Interface, fledgedclientset clientset.Interface) (*Controller, coreinformers.NodeInformer, fledgedinformers.ImageCacheInformer) { + kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeclientset, noResyncPeriodFunc()) + fledgedInformerFactory := informers.NewSharedInformerFactory(fledgedclientset, noResyncPeriodFunc()) + nodeInformer := kubeInformerFactory.Core().V1().Nodes() + imagecacheInformer := fledgedInformerFactory.Fledged().V1alpha1().ImageCaches() + imageCacheRefreshFrequency := time.Second * 0 + imagePullDeadlineDuration := time.Second * 5 + dockerClientImage := "senthilrch/fledged-docker-client:latest" + imagePullPolicy := "IfNotPresent" + + /* startInformers := true + if startInformers { + stopCh := make(chan struct{}) + defer close(stopCh) + kubeInformerFactory.Start(stopCh) + fledgedInformerFactory.Start(stopCh) + } */ + + controller := NewController(kubeclientset, fledgedclientset, fledgedNameSpace, nodeInformer, imagecacheInformer, + imageCacheRefreshFrequency, imagePullDeadlineDuration, dockerClientImage, imagePullPolicy) + controller.nodesSynced = func() bool { return true } + controller.imageCachesSynced = func() bool { return true } + return controller, nodeInformer, imagecacheInformer +} + +func TestPreFlightChecks(t *testing.T) { + tests := []struct { + name string + jobList *batchv1.JobList + jobListError error + jobDeleteError error + imageCacheList *fledgedv1alpha1.ImageCacheList + imageCacheListError error + imageCacheUpdateError error + expectErr bool + errorString string + }{ + { + name: "#1: No dangling jobs. No imagecaches", + jobList: &batchv1.JobList{Items: []batchv1.Job{}}, + jobListError: nil, + jobDeleteError: nil, + imageCacheList: &fledgedv1alpha1.ImageCacheList{Items: []fledgedv1alpha1.ImageCache{}}, + imageCacheListError: nil, + imageCacheUpdateError: nil, + expectErr: false, + errorString: "", + }, + { + name: "#2: No dangling jobs. No dangling imagecaches", + jobList: &batchv1.JobList{Items: []batchv1.Job{}}, + jobListError: nil, + jobDeleteError: nil, + imageCacheList: &fledgedv1alpha1.ImageCacheList{ + Items: []fledgedv1alpha1.ImageCache{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, + }, + }, + }, + }, + imageCacheListError: nil, + imageCacheUpdateError: nil, + expectErr: false, + errorString: "", + }, + { + name: "#3: One dangling job. One dangling image cache. Successful list and delete", + //imageCache: nil, + jobList: &batchv1.JobList{ + Items: []batchv1.Job{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + }, + }, + }, + jobListError: nil, + jobDeleteError: nil, + imageCacheList: &fledgedv1alpha1.ImageCacheList{ + Items: []fledgedv1alpha1.ImageCache{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, + }, + }, + }, + }, + imageCacheListError: nil, + imageCacheUpdateError: nil, + expectErr: false, + errorString: "", + }, + { + name: "#4: Unsuccessful listing of jobs", + jobList: nil, + jobListError: fmt.Errorf("fake error"), + jobDeleteError: nil, + expectErr: true, + errorString: "Internal error occurred: fake error", + }, + { + name: "#5: Unsuccessful listing of imagecaches", + jobList: &batchv1.JobList{Items: []batchv1.Job{}}, + jobListError: nil, + jobDeleteError: nil, + imageCacheList: nil, + imageCacheListError: fmt.Errorf("fake error"), + imageCacheUpdateError: nil, + expectErr: true, + errorString: "Internal error occurred: fake error", + }, + { + name: "#6: One dangling job. Successful list. Unsuccessful delete", + jobList: &batchv1.JobList{ + Items: []batchv1.Job{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + }, + }, + }, + jobListError: nil, + jobDeleteError: fmt.Errorf("fake error"), + expectErr: true, + errorString: "Internal error occurred: fake error", + }, + { + name: "#7: One dangling image cache. Successful list. Unsuccessful delete", + jobList: &batchv1.JobList{Items: []batchv1.Job{}}, + jobListError: nil, + jobDeleteError: nil, + imageCacheList: &fledgedv1alpha1.ImageCacheList{ + Items: []fledgedv1alpha1.ImageCache{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, + }, + }, + }, + }, + imageCacheListError: nil, + imageCacheUpdateError: fmt.Errorf("fake error"), + expectErr: true, + errorString: "Internal error occurred: fake error", + }, + } + for _, test := range tests { + fakekubeclientset := &fakeclientset.Clientset{} + fakefledgedclientset := &fledgedclientsetfake.Clientset{} + if test.jobListError != nil { + listError := apierrors.NewInternalError(test.jobListError) + fakekubeclientset.AddReactor("list", "jobs", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, listError + }) + } else { + fakekubeclientset.AddReactor("list", "jobs", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, test.jobList, nil + }) + } + if test.jobDeleteError != nil { + deleteError := apierrors.NewInternalError(test.jobDeleteError) + fakekubeclientset.AddReactor("delete", "jobs", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, deleteError + }) + } else { + fakekubeclientset.AddReactor("delete", "jobs", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, nil + }) + } + + if test.imageCacheListError != nil { + listError := apierrors.NewInternalError(test.imageCacheListError) + fakefledgedclientset.AddReactor("list", "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, listError + }) + } else { + fakefledgedclientset.AddReactor("list", "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, test.imageCacheList, nil + }) + } + if test.imageCacheUpdateError != nil { + updateError := apierrors.NewInternalError(test.imageCacheUpdateError) + fakefledgedclientset.AddReactor("update", "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, updateError + }) + } else { + fakefledgedclientset.AddReactor("update", "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, nil + }) + } + + controller, _, _ := newTestController(fakekubeclientset, fakefledgedclientset) + + err := controller.PreFlightChecks() + if test.expectErr { + if !(err != nil && strings.HasPrefix(err.Error(), test.errorString)) { + t.Errorf("Test: %s failed", test.name) + } + } else { + if err != nil { + t.Errorf("Test: %s failed. err received = %s", test.name, err.Error()) + } + } + } + t.Logf("%d tests passed", len(tests)) +} + +func TestRunRefreshWorker(t *testing.T) { + tests := []struct { + name string + imageCacheList *fledgedv1alpha1.ImageCacheList + imageCacheListError error + workqueueItems int + }{ + { + name: "#1: Do not refresh if status is not yet updated", + imageCacheList: &fledgedv1alpha1.ImageCacheList{ + Items: []fledgedv1alpha1.ImageCache{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + }, + }, + }, + imageCacheListError: nil, + workqueueItems: 0, + }, + { + name: "#2: Do not refresh if image cache is already under processing", + imageCacheList: &fledgedv1alpha1.ImageCacheList{ + Items: []fledgedv1alpha1.ImageCache{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, + }, + }, + }, + }, + imageCacheListError: nil, + workqueueItems: 0, + }, + { + name: "#3: Do not refresh image cache if cache spec validation failed", + imageCacheList: &fledgedv1alpha1.ImageCacheList{ + Items: []fledgedv1alpha1.ImageCache{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusFailed, + Reason: fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, + }, + }, + }, + }, + imageCacheListError: nil, + workqueueItems: 0, + }, + { + name: "#4: Do not refresh if image cache has been purged", + imageCacheList: &fledgedv1alpha1.ImageCacheList{ + Items: []fledgedv1alpha1.ImageCache{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Reason: fledgedv1alpha1.ImageCacheReasonImageCachePurge, + }, + }, + }, + }, + imageCacheListError: nil, + workqueueItems: 0, + }, + { + name: "#5: Successfully queued 1 imagecache for refresh", + imageCacheList: &fledgedv1alpha1.ImageCacheList{ + Items: []fledgedv1alpha1.ImageCache{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, + }, + }, + }, + }, + imageCacheListError: nil, + workqueueItems: 1, + }, + { + name: "#6: Successfully queued 2 imagecaches for refresh", + imageCacheList: &fledgedv1alpha1.ImageCacheList{ + Items: []fledgedv1alpha1.ImageCache{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Namespace: "kube-fledged", + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusFailed, + }, + }, + }, + }, + imageCacheListError: nil, + workqueueItems: 2, + }, + { + name: "#7: No imagecaches to refresh", + imageCacheList: nil, + imageCacheListError: nil, + workqueueItems: 0, + }, + } + + for _, test := range tests { + if test.workqueueItems > 0 { + //TODO: How to check if workqueue contains the added item? + continue + } + fakekubeclientset := &fakeclientset.Clientset{} + fakefledgedclientset := &fledgedclientsetfake.Clientset{} + + controller, _, imagecacheInformer := newTestController(fakekubeclientset, fakefledgedclientset) + if test.imageCacheList != nil && len(test.imageCacheList.Items) > 0 { + for _, imagecache := range test.imageCacheList.Items { + imagecacheInformer.Informer().GetIndexer().Add(&imagecache) + } + } + controller.runRefreshWorker() + if test.workqueueItems == controller.workqueue.Len() { + } else { + t.Errorf("Test: %s failed: expected %d, actual %d", test.name, test.workqueueItems, controller.workqueue.Len()) + } + } +} + +func TestSyncHandler(t *testing.T) { + type ActionReaction struct { + action string + reaction string + } + now := metav1.Now() + defaultImageCache := fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + } + defaultNodeList := &corev1.NodeList{ + Items: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakenode", + Labels: map[string]string{"kubernetes.io/hostname": "bar"}, + }, + }, + }, + } + + tests := []struct { + name string + imageCache fledgedv1alpha1.ImageCache + wqKey images.WorkQueueKey + nodeList *corev1.NodeList + expectedActions []ActionReaction + expectErr bool + expectedErrString string + }{ + { + name: "#1: Invalid imagecache resource key", + wqKey: images.WorkQueueKey{ + ObjKey: "foo/bar/car", + }, + expectErr: true, + expectedErrString: "unexpected key format", + }, + { + name: "#2: Create - Invalid imagecache spec (no images specified)", + imageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{}, + }, + }, + }, + }, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheCreate, + }, + expectedActions: []ActionReaction{{action: "update", reaction: ""}}, + expectErr: true, + expectedErrString: "No images specified within image list", + }, + { + name: "#3: Update - Old imagecache pointer is nil", + imageCache: defaultImageCache, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheUpdate, + OldImageCache: nil, + }, + nodeList: defaultNodeList, + expectedActions: []ActionReaction{{action: "update", reaction: ""}}, + expectErr: true, + expectedErrString: "OldImageCacheNotFound", + }, + { + name: "#4: Update - No. of imagelists not equal", + imageCache: defaultImageCache, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheUpdate, + OldImageCache: &fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + { + Images: []string{"bar"}, + }, + }, + }, + }, + }, + nodeList: defaultNodeList, + expectedActions: []ActionReaction{{action: "update", reaction: ""}}, + expectErr: true, + expectedErrString: "CacheSpecValidationFailed", + }, + { + name: "#5: Update - Change in NodeSelectors", + imageCache: defaultImageCache, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheUpdate, + OldImageCache: &fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + NodeSelector: map[string]string{"foo": "bar"}, + }, + }, + }, + }, + }, + nodeList: defaultNodeList, + expectedActions: []ActionReaction{{action: "update", reaction: ""}}, + expectErr: true, + expectedErrString: "CacheSpecValidationFailed", + }, + { + name: "#6: Refresh - Update status to processing", + imageCache: defaultImageCache, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheRefresh, + }, + nodeList: defaultNodeList, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: "fake error"}, + }, + expectErr: true, + expectedErrString: "Internal error occurred: fake error", + }, + { + name: "#7: StatusUpdate - Successful Refresh", + imageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + StartTime: &now, + Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, + Reason: fledgedv1alpha1.ImageCacheReasonImageCacheRefresh, + }, + }, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheStatusUpdate, + Status: &map[string]images.ImageWorkResult{ + "job1": { + Status: images.ImageWorkResultStatusSucceeded, + ImageWorkRequest: images.ImageWorkRequest{ + WorkType: images.ImageCacheRefresh, + Node: &node, + }, + }, + }, + }, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: ""}, + }, + expectErr: false, + expectedErrString: "", + }, + { + name: "#8: Purge - Update status to processing", + imageCache: defaultImageCache, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCachePurge, + }, + nodeList: defaultNodeList, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: "fake error"}, + }, + expectErr: true, + expectedErrString: "Internal error occurred: fake error", + }, + { + name: "#9: Purge - Successful purge", + imageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Reason: fledgedv1alpha1.ImageCacheReasonImageCachePurge, + }, + }, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheRefresh, + }, + nodeList: defaultNodeList, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: ""}, + }, + expectErr: false, + expectedErrString: "", + }, + { + name: "#10: Create - Successfully firing imagepull requests", + imageCache: defaultImageCache, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheCreate, + }, + nodeList: defaultNodeList, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: ""}, + }, + expectErr: false, + expectedErrString: "", + }, + { + name: "#11: Update - Successfully firing imagepull & imagedelete requests", + imageCache: defaultImageCache, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheUpdate, + OldImageCache: &fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo", "bar"}, + }, + }, + }, + }, + }, + nodeList: defaultNodeList, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: ""}, + }, + expectErr: false, + expectedErrString: "", + }, + { + name: "#12: StatusUpdate - ImagesPulledSuccessfully", + imageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + StartTime: &now, + }, + }, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheStatusUpdate, + Status: &map[string]images.ImageWorkResult{ + "job1": { + Status: images.ImageWorkResultStatusSucceeded, + ImageWorkRequest: images.ImageWorkRequest{ + WorkType: images.ImageCacheCreate, + Node: &node, + }, + }, + }, + }, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: ""}, + }, + expectErr: false, + expectedErrString: "", + }, + { + name: "#13: StatusUpdate - ImagesDeletedSuccessfully", + imageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + StartTime: &now, + Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, + Reason: fledgedv1alpha1.ImageCacheReasonImageCachePurge, + }, + }, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheStatusUpdate, + Status: &map[string]images.ImageWorkResult{ + "job1": { + Status: images.ImageWorkResultStatusSucceeded, + ImageWorkRequest: images.ImageWorkRequest{ + WorkType: images.ImageCachePurge, + Node: &node, + }, + }, + }, + }, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: ""}, + }, + expectErr: false, + expectedErrString: "", + }, + { + name: "#14: StatusUpdate - ImagePullFailedForSomeImages", + imageCache: defaultImageCache, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheStatusUpdate, + Status: &map[string]images.ImageWorkResult{ + "job1": { + Status: images.ImageWorkResultStatusFailed, + ImageWorkRequest: images.ImageWorkRequest{ + WorkType: images.ImageCacheCreate, + Node: &node, + }, + }, + }, + }, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: ""}, + }, + expectErr: false, + expectedErrString: "", + }, + { + name: "#15: StatusUpdate - ImageDeleteFailedForSomeImages", + imageCache: defaultImageCache, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheStatusUpdate, + Status: &map[string]images.ImageWorkResult{ + "job1": { + Status: images.ImageWorkResultStatusFailed, + ImageWorkRequest: images.ImageWorkRequest{ + WorkType: images.ImageCachePurge, + Node: &node, + }, + }, + }, + }, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: ""}, + }, + expectErr: false, + expectedErrString: "", + }, + } + + for _, test := range tests { + fakekubeclientset := &fakeclientset.Clientset{} + fakefledgedclientset := &fledgedclientsetfake.Clientset{} + for _, ar := range test.expectedActions { + if ar.reaction != "" { + apiError := apierrors.NewInternalError(fmt.Errorf(ar.reaction)) + fakefledgedclientset.AddReactor(ar.action, "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, apiError + }) + } + fakefledgedclientset.AddReactor(ar.action, "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, &test.imageCache, nil + }) + } + + controller, nodeInformer, imagecacheInformer := newTestController(fakekubeclientset, fakefledgedclientset) + if test.nodeList != nil && len(test.nodeList.Items) > 0 { + for _, node := range test.nodeList.Items { + nodeInformer.Informer().GetIndexer().Add(&node) + } + } + imagecacheInformer.Informer().GetIndexer().Add(&test.imageCache) + err := controller.syncHandler(test.wqKey) + if test.expectErr { + if err == nil { + t.Errorf("Test: %s failed: expectedError=%s, actualError=nil", test.name, test.expectedErrString) + } + if err != nil && !strings.HasPrefix(err.Error(), test.expectedErrString) { + t.Errorf("Test: %s failed: expectedError=%s, actualError=%s", test.name, test.expectedErrString, err.Error()) + } + } else if err != nil { + t.Errorf("Test: %s failed. expectedError=nil, actualError=%s", test.name, err.Error()) + } + } + t.Logf("%d tests passed", len(tests)) +} + +func TestEnqueueImageCache(t *testing.T) { + //now := metav1.Now() + //nowplus5s := metav1.NewTime(time.Now().Add(time.Second * 5)) + defaultImageCache := fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + } + tests := []struct { + name string + workType images.WorkType + oldImageCache fledgedv1alpha1.ImageCache + newImageCache fledgedv1alpha1.ImageCache + expectedResult bool + }{ + { + name: "#1: Create - Imagecache queued successfully", + workType: images.ImageCacheCreate, + newImageCache: defaultImageCache, + expectedResult: true, + }, + { + name: "#2: Create - Imagecache with Status field, so no queueing", + workType: images.ImageCacheCreate, + newImageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, + }, + }, + expectedResult: false, + }, + { + name: "#3: Update - Imagecache purge. Successful queueing", + workType: images.ImageCacheUpdate, + oldImageCache: defaultImageCache, + newImageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + Annotations: map[string]string{imageCachePurgeAnnotationKey: ""}, + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, + }, + }, + expectedResult: true, + }, + { + name: "#4: Update - No change in Spec. Unsuccessful queueing", + workType: images.ImageCacheUpdate, + oldImageCache: defaultImageCache, + newImageCache: defaultImageCache, + expectedResult: false, + }, + { + name: "#5: Update - Status processing. Unsuccessful queueing", + workType: images.ImageCacheUpdate, + oldImageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, + }, + }, + expectedResult: false, + }, + { + name: "#6: Update - Successful queueing", + workType: images.ImageCacheUpdate, + oldImageCache: defaultImageCache, + newImageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo", "bar"}, + }, + }, + }, + }, + expectedResult: true, + }, + { + name: "#7: Delete - Unsuccessful queueing", + workType: images.ImageCacheDelete, + expectedResult: false, + }, + { + name: "#8: Refresh - Successful queueing", + workType: images.ImageCacheRefresh, + oldImageCache: defaultImageCache, + expectedResult: true, + }, + { + name: "#9: Update - CacheSpec restoration", + workType: images.ImageCacheUpdate, + oldImageCache: defaultImageCache, + newImageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + Annotations: map[string]string{ + fledgedCacheSpecValidationKey: "failed", + }, + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo", "bar"}, + }, + }, + }, + }, + expectedResult: false, + }, + { + name: "#10: Update - Imagecache refresh. Successful queueing", + workType: images.ImageCacheUpdate, + oldImageCache: defaultImageCache, + newImageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + Annotations: map[string]string{imageCacheRefreshAnnotationKey: ""}, + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, + }, + }, + expectedResult: true, + }, + } + + for _, test := range tests { + fakekubeclientset := &fakeclientset.Clientset{} + fakefledgedclientset := &fledgedclientsetfake.Clientset{} + controller, _, _ := newTestController(fakekubeclientset, fakefledgedclientset) + result := controller.enqueueImageCache(test.workType, &test.oldImageCache, &test.newImageCache) + if result != test.expectedResult { + t.Errorf("Test %s failed: expected=%t, actual=%t", test.name, test.expectedResult, result) + } + } +} + +func TestProcessNextWorkItem(t *testing.T) { + type ActionReaction struct { + action string + reaction string + } + defaultImageCache := fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + } + + tests := []struct { + name string + imageCache fledgedv1alpha1.ImageCache + wqKey images.WorkQueueKey + expectedActions []ActionReaction + expectErr bool + expectedErrString string + }{ + { + name: "#1: StatusUpdate - ImageDeleteFailedForSomeImages", + imageCache: defaultImageCache, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheStatusUpdate, + Status: &map[string]images.ImageWorkResult{ + "job1": { + Status: images.ImageWorkResultStatusFailed, + ImageWorkRequest: images.ImageWorkRequest{ + WorkType: images.ImageCachePurge, + Node: &node, + }, + }, + }, + }, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: ""}, + }, + expectErr: false, + expectedErrString: "", + }, + { + name: "#2: Create - Invalid imagecache spec (no images specified)", + imageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{}, + }, + }, + }, + }, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheCreate, + }, + expectedActions: []ActionReaction{{action: "update", reaction: ""}}, + expectErr: false, + expectedErrString: "No images specified within image list", + }, + { + name: "#3: Unexpected type in workqueue", + expectErr: false, + expectedErrString: "Unexpected type in workqueue", + }, + } + + for _, test := range tests { + fakekubeclientset := &fakeclientset.Clientset{} + fakefledgedclientset := &fledgedclientsetfake.Clientset{} + for _, ar := range test.expectedActions { + if ar.reaction != "" { + apiError := apierrors.NewInternalError(fmt.Errorf(ar.reaction)) + fakefledgedclientset.AddReactor(ar.action, "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, apiError + }) + } + fakefledgedclientset.AddReactor(ar.action, "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, &test.imageCache, nil + }) + } + + controller, _, imagecacheInformer := newTestController(fakekubeclientset, fakefledgedclientset) + imagecacheInformer.Informer().GetIndexer().Add(&test.imageCache) + if test.expectedErrString == "Unexpected type in workqueue" { + controller.workqueue.Add(struct{}{}) + } + controller.workqueue.Add(test.wqKey) + controller.processNextWorkItem() + var err error + if test.expectErr { + if err == nil { + t.Errorf("Test: %s failed: expectedError=%s, actualError=nil", test.name, test.expectedErrString) + } + if err != nil && !strings.HasPrefix(err.Error(), test.expectedErrString) { + t.Errorf("Test: %s failed: expectedError=%s, actualError=%s", test.name, test.expectedErrString, err.Error()) + } + } else if err != nil { + t.Errorf("Test: %s failed. expectedError=nil, actualError=%s", test.name, err.Error()) + } + } + t.Logf("%d tests passed", len(tests)) +} diff --git a/cmd/fledged.go b/cmd/fledged/main.go similarity index 91% rename from cmd/fledged.go rename to cmd/fledged/main.go index 75666658..a112ab54 100644 --- a/cmd/fledged.go +++ b/cmd/fledged/main.go @@ -28,11 +28,10 @@ import ( // Uncomment the following line to load the gcp plugin (only required to authenticate against GKE clusters). // _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" - "github.com/senthilrch/kube-fledged/cmd/app" + "github.com/senthilrch/kube-fledged/cmd/fledged/app" clientset "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned" informers "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions" "github.com/senthilrch/kube-fledged/pkg/signals" - "github.com/senthilrch/kube-fledged/pkg/webhook" ) var ( @@ -85,11 +84,6 @@ func main() { if err = controller.Run(1, stopCh); err != nil { glog.Fatalf("Error running controller: %s", err.Error()) } - - if err := webhook.CreateAndStartWebHookServer(stopCh, webhookServerPort); err != nil { - glog.Fatalf("Error creating webhook server: %s", err.Error()) - } - } func init() { @@ -100,5 +94,4 @@ func init() { if fledgedNameSpace = os.Getenv("KUBEFLEDGED_NAMESPACE"); fledgedNameSpace == "" { fledgedNameSpace = "kube-fledged" } - flag.IntVar(&webhookServerPort, "webhook-server-port", 3443, "Webhook server port.") } diff --git a/cmd/webhook-server/main.go b/cmd/webhook-server/main.go new file mode 100644 index 00000000..aa3040ea --- /dev/null +++ b/cmd/webhook-server/main.go @@ -0,0 +1,286 @@ +/* +Copyright 2018 The kube-fledged authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "crypto/tls" + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "net/http" + + "github.com/senthilrch/kube-fledged/pkg/webhook" + + admissionv1 "k8s.io/api/admission/v1" + v1 "k8s.io/api/admission/v1" + "k8s.io/api/admission/v1beta1" + admissionv1beta1 "k8s.io/api/admission/v1beta1" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/klog" + // TODO: try this library to see if it generates correct json patch + // https://github.com/mattbaird/jsonpatch +) + +var scheme = runtime.NewScheme() +var codecs = serializer.NewCodecFactory(scheme) + +func addToScheme(scheme *runtime.Scheme) { + utilruntime.Must(corev1.AddToScheme(scheme)) + utilruntime.Must(admissionv1beta1.AddToScheme(scheme)) + utilruntime.Must(admissionregistrationv1beta1.AddToScheme(scheme)) + utilruntime.Must(admissionv1.AddToScheme(scheme)) + utilruntime.Must(admissionregistrationv1.AddToScheme(scheme)) +} + +func init() { + addToScheme(scheme) +} + +var ( + certFile string + keyFile string + port int +) + +func init() { + flag.StringVar(&certFile, "cert-file", "", "File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated after server cert).") + flag.StringVar(&keyFile, "key-file", "", "File containing the default x509 private key matching --cert-file.") + flag.IntVar(&port, "port", 443, "Secure port that the webhook server listens on") +} + +// admitv1beta1Func handles a v1beta1 admission +type admitv1beta1Func func(v1beta1.AdmissionReview) *v1beta1.AdmissionResponse + +// admitv1beta1Func handles a v1 admission +type admitv1Func func(v1.AdmissionReview) *v1.AdmissionResponse + +// admitHandler is a handler, for both validators and mutators, that supports multiple admission review versions +type admitHandler struct { + v1beta1 admitv1beta1Func + v1 admitv1Func +} + +// Config contains the server (the webhook) cert and key. +type Config struct { + CertFile string + KeyFile string +} + +func configTLS(config Config) *tls.Config { + sCert, err := tls.LoadX509KeyPair(config.CertFile, config.KeyFile) + if err != nil { + klog.Fatal(err) + } + return &tls.Config{ + Certificates: []tls.Certificate{sCert}, + // TODO: uses mutual tls after we agree on what cert the apiserver should use. + // ClientAuth: tls.RequireAndVerifyClientCert, + } +} + +func newDelegateToV1AdmitHandler(f admitv1Func) admitHandler { + return admitHandler{ + v1beta1: delegateV1beta1AdmitToV1(f), + v1: f, + } +} + +func delegateV1beta1AdmitToV1(f admitv1Func) admitv1beta1Func { + return func(review v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { + in := v1.AdmissionReview{Request: convertAdmissionRequestToV1(review.Request)} + out := f(in) + return convertAdmissionResponseToV1beta1(out) + } +} + +// serve handles the http portion of a request prior to handing to an admit +// function +func serve(w http.ResponseWriter, r *http.Request, admit admitHandler) { + var body []byte + if r.Body != nil { + if data, err := ioutil.ReadAll(r.Body); err == nil { + body = data + } + } + + // verify the content type is accurate + contentType := r.Header.Get("Content-Type") + if contentType != "application/json" { + klog.Errorf("contentType=%s, expect application/json", contentType) + return + } + + klog.V(2).Info(fmt.Sprintf("handling request: %s", body)) + + deserializer := codecs.UniversalDeserializer() + obj, gvk, err := deserializer.Decode(body, nil, nil) + if err != nil { + msg := fmt.Sprintf("Request could not be decoded: %v", err) + klog.Error(msg) + http.Error(w, msg, http.StatusBadRequest) + return + } + + var responseObj runtime.Object + switch *gvk { + case v1beta1.SchemeGroupVersion.WithKind("AdmissionReview"): + requestedAdmissionReview, ok := obj.(*v1beta1.AdmissionReview) + if !ok { + klog.Errorf("Expected v1beta1.AdmissionReview but got: %T", obj) + return + } + responseAdmissionReview := &v1beta1.AdmissionReview{} + responseAdmissionReview.SetGroupVersionKind(*gvk) + responseAdmissionReview.Response = admit.v1beta1(*requestedAdmissionReview) + responseAdmissionReview.Response.UID = requestedAdmissionReview.Request.UID + responseObj = responseAdmissionReview + case v1.SchemeGroupVersion.WithKind("AdmissionReview"): + requestedAdmissionReview, ok := obj.(*v1.AdmissionReview) + if !ok { + klog.Errorf("Expected v1.AdmissionReview but got: %T", obj) + return + } + responseAdmissionReview := &v1.AdmissionReview{} + responseAdmissionReview.SetGroupVersionKind(*gvk) + responseAdmissionReview.Response = admit.v1(*requestedAdmissionReview) + responseAdmissionReview.Response.UID = requestedAdmissionReview.Request.UID + responseObj = responseAdmissionReview + default: + msg := fmt.Sprintf("Unsupported group version kind: %v", gvk) + klog.Error(msg) + http.Error(w, msg, http.StatusBadRequest) + return + } + + klog.V(2).Info(fmt.Sprintf("sending response: %v", responseObj)) + respBytes, err := json.Marshal(responseObj) + if err != nil { + klog.Error(err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + if _, err := w.Write(respBytes); err != nil { + klog.Error(err) + } +} + +func convertAdmissionRequestToV1(r *v1beta1.AdmissionRequest) *v1.AdmissionRequest { + return &v1.AdmissionRequest{ + Kind: r.Kind, + Namespace: r.Namespace, + Name: r.Name, + Object: r.Object, + Resource: r.Resource, + Operation: v1.Operation(r.Operation), + UID: r.UID, + DryRun: r.DryRun, + OldObject: r.OldObject, + Options: r.Options, + RequestKind: r.RequestKind, + RequestResource: r.RequestResource, + RequestSubResource: r.RequestSubResource, + SubResource: r.SubResource, + UserInfo: r.UserInfo, + } +} + +func convertAdmissionRequestToV1beta1(r *v1.AdmissionRequest) *v1beta1.AdmissionRequest { + return &v1beta1.AdmissionRequest{ + Kind: r.Kind, + Namespace: r.Namespace, + Name: r.Name, + Object: r.Object, + Resource: r.Resource, + Operation: v1beta1.Operation(r.Operation), + UID: r.UID, + DryRun: r.DryRun, + OldObject: r.OldObject, + Options: r.Options, + RequestKind: r.RequestKind, + RequestResource: r.RequestResource, + RequestSubResource: r.RequestSubResource, + SubResource: r.SubResource, + UserInfo: r.UserInfo, + } +} + +func convertAdmissionResponseToV1(r *v1beta1.AdmissionResponse) *v1.AdmissionResponse { + var pt *v1.PatchType + if r.PatchType != nil { + t := v1.PatchType(*r.PatchType) + pt = &t + } + return &v1.AdmissionResponse{ + UID: r.UID, + Allowed: r.Allowed, + AuditAnnotations: r.AuditAnnotations, + Patch: r.Patch, + PatchType: pt, + Result: r.Result, + } +} + +func convertAdmissionResponseToV1beta1(r *v1.AdmissionResponse) *v1beta1.AdmissionResponse { + var pt *v1beta1.PatchType + if r.PatchType != nil { + t := v1beta1.PatchType(*r.PatchType) + pt = &t + } + return &v1beta1.AdmissionResponse{ + UID: r.UID, + Allowed: r.Allowed, + AuditAnnotations: r.AuditAnnotations, + Patch: r.Patch, + PatchType: pt, + Result: r.Result, + } +} + +func validateImageCache(w http.ResponseWriter, r *http.Request) { + serve(w, r, newDelegateToV1AdmitHandler(webhook.ValidateImageCache)) +} + +func mutateImageCache(w http.ResponseWriter, r *http.Request) { + serve(w, r, newDelegateToV1AdmitHandler(webhook.MutateImageCache)) +} + +func main() { + flag.Parse() + config := Config{ + CertFile: certFile, + KeyFile: keyFile, + } + + http.HandleFunc("/validate-image-cache", validateImageCache) + http.HandleFunc("/mutate-image-cache", mutateImageCache) + http.HandleFunc("/readyz", func(w http.ResponseWriter, req *http.Request) { w.Write([]byte("ok")) }) + server := &http.Server{ + Addr: fmt.Sprintf(":%d", port), + TLSConfig: configTLS(config), + } + err := server.ListenAndServeTLS("", "") + if err != nil { + panic(err) + } +} diff --git a/deploy/kubefledged-deployment.yaml b/deploy/kubefledged-deployment.yaml index befeb536..e6575829 100644 --- a/deploy/kubefledged-deployment.yaml +++ b/deploy/kubefledged-deployment.yaml @@ -1,34 +1,34 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: kubefledged - namespace: kube-fledged -spec: - replicas: 1 - selector: - matchLabels: - app: kubefledged - template: - metadata: - labels: - app: kubefledged - spec: - containers: - - image: senthilrch/fledged:v0.7.0 - command: ["/opt/bin/fledged"] - args: - - "--stderrthreshold=INFO" - - "--image-pull-deadline-duration=5m" - - "--image-cache-refresh-frequency=15m" - - "--docker-client-image=senthilrch/fledged-docker-client:v0.7.0" - - "--image-pull-policy=IfNotPresent" - imagePullPolicy: Always - name: fledged - env: - - name: KUBEFLEDGED_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - serviceAccountName: kubefledged - - +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kubefledged + namespace: kube-fledged +spec: + replicas: 1 + selector: + matchLabels: + app: kubefledged + template: + metadata: + labels: + app: kubefledged + spec: + containers: + - image: senthilrch/fledged:v0.7.0 + command: ["/opt/bin/fledged"] + args: + - "--stderrthreshold=INFO" + - "--image-pull-deadline-duration=5m" + - "--image-cache-refresh-frequency=15m" + - "--docker-client-image=senthilrch/fledged-docker-client:v0.7.0" + - "--image-pull-policy=IfNotPresent" + imagePullPolicy: Always + name: fledged + env: + - name: KUBEFLEDGED_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + serviceAccountName: kubefledged + + diff --git a/deploy/kubefledged-validatingwebhook.yaml b/deploy/kubefledged-validatingwebhook.yaml new file mode 100644 index 00000000..bb2bee2d --- /dev/null +++ b/deploy/kubefledged-validatingwebhook.yaml @@ -0,0 +1,21 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: kubefledged +webhooks: + - name: validate-image-cache + admissionReviewVersions: ["v1"] + timeoutSeconds: 1 + failurePolicy: Fail + clientConfig: + service: + namespace: kube-fledged + name: kubefledged-webhook-svc + path: "/validate-image-cache" + port: 443 + rules: + - operations: ["CREATE", "UPDATE"] + apiGroups: ["fledged.k8s.io"] + apiVersions: ["v1alpha1"] + resources: ["imagecaches"] + scope: "Namespaced" \ No newline at end of file diff --git a/go.mod b/go.mod index ead45cdd..87e19f30 100644 --- a/go.mod +++ b/go.mod @@ -12,4 +12,5 @@ require ( k8s.io/apiserver v0.17.2 k8s.io/client-go v0.17.2 k8s.io/code-generator v0.17.2 // indirect + k8s.io/klog v1.0.0 ) diff --git a/go.sum b/go.sum index 579f7bdc..2de5262f 100644 --- a/go.sum +++ b/go.sum @@ -129,6 +129,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -200,6 +201,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= diff --git a/pkg/webhook/imagecache.go b/pkg/webhook/imagecache.go new file mode 100644 index 00000000..6e49195e --- /dev/null +++ b/pkg/webhook/imagecache.go @@ -0,0 +1,113 @@ +/* +Copyright 2018 The kube-fledged authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "encoding/json" + + v1 "k8s.io/api/admission/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" +) + +const ( + customResourcePatch1 string = `[ + { "op": "add", "path": "/data/mutation-stage-1", "value": "yes" } + ]` + customResourcePatch2 string = `[ + { "op": "add", "path": "/data/mutation-stage-2", "value": "yes" } + ]` +) + +// MutateImageCache modifies image cache resource +func MutateImageCache(ar v1.AdmissionReview) *v1.AdmissionResponse { + klog.V(2).Info("mutating custom resource") + cr := struct { + metav1.ObjectMeta + Data map[string]string + }{} + + raw := ar.Request.Object.Raw + err := json.Unmarshal(raw, &cr) + if err != nil { + klog.Error(err) + return toV1AdmissionResponse(err) + } + + reviewResponse := v1.AdmissionResponse{} + reviewResponse.Allowed = true + + if cr.Data["mutation-start"] == "yes" { + reviewResponse.Patch = []byte(customResourcePatch1) + } + if cr.Data["mutation-stage-1"] == "yes" { + reviewResponse.Patch = []byte(customResourcePatch2) + } + if len(reviewResponse.Patch) != 0 { + pt := v1.PatchTypeJSONPatch + reviewResponse.PatchType = &pt + } + return &reviewResponse +} + +// ValidateImageCache validates image cache resource +func ValidateImageCache(ar v1.AdmissionReview) *v1.AdmissionResponse { + klog.V(2).Info("admitting custom resource") + cr := struct { + metav1.ObjectMeta + Data map[string]string + }{} + + var raw []byte + if ar.Request.Operation == v1.Delete { + raw = ar.Request.OldObject.Raw + } else { + raw = ar.Request.Object.Raw + } + err := json.Unmarshal(raw, &cr) + if err != nil { + klog.Error(err) + return toV1AdmissionResponse(err) + } + + reviewResponse := v1.AdmissionResponse{} + reviewResponse.Allowed = true + for k, v := range cr.Data { + if k == "webhook-e2e-test" && v == "webhook-disallow" && + (ar.Request.Operation == v1.Create || ar.Request.Operation == v1.Update) { + reviewResponse.Allowed = false + reviewResponse.Result = &metav1.Status{ + Reason: "the custom resource contains unwanted data", + } + } + if k == "webhook-e2e-test" && v == "webhook-nondeletable" && ar.Request.Operation == v1.Delete { + reviewResponse.Allowed = false + reviewResponse.Result = &metav1.Status{ + Reason: "the custom resource cannot be deleted because it contains unwanted key and value", + } + } + } + return &reviewResponse +} + +func toV1AdmissionResponse(err error) *v1.AdmissionResponse { + return &v1.AdmissionResponse{ + Result: &metav1.Status{ + Message: err.Error(), + }, + } +} diff --git a/pkg/webhook/mutate.go b/pkg/webhook/mutate.go deleted file mode 100644 index a1ce5b2f..00000000 --- a/pkg/webhook/mutate.go +++ /dev/null @@ -1,161 +0,0 @@ -/* -Copyright 2018 The kube-fledged authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package webhook - -import ( - "encoding/json" - "strings" - - "github.com/golang/glog" - "k8s.io/api/admission/v1beta1" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// main mutation process -func (whsvr *Server) mutate(ar *v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { - req := ar.Request - var ( - availableLabels, availableAnnotations map[string]string - objectMeta *metav1.ObjectMeta - resourceNamespace, resourceName string - ) - - glog.Infof("AdmissionReview for Kind=%v, Namespace=%v Name=%v (%v) UID=%v patchOperation=%v UserInfo=%v", - req.Kind, req.Namespace, req.Name, resourceName, req.UID, req.Operation, req.UserInfo) - - switch req.Kind.Kind { - case "Deployment": - var deployment appsv1.Deployment - if err := json.Unmarshal(req.Object.Raw, &deployment); err != nil { - glog.Errorf("Could not unmarshal raw object: %v", err) - return &v1beta1.AdmissionResponse{ - Result: &metav1.Status{ - Message: err.Error(), - }, - } - } - - resourceName, resourceNamespace, objectMeta = deployment.Name, deployment.Namespace, &deployment.ObjectMeta - availableLabels = deployment.Labels - - case "Service": - var service corev1.Service - if err := json.Unmarshal(req.Object.Raw, &service); err != nil { - glog.Errorf("Could not unmarshal raw object: %v", err) - return &v1beta1.AdmissionResponse{ - Result: &metav1.Status{ - Message: err.Error(), - }, - } - } - - resourceName, resourceNamespace, objectMeta = service.Name, service.Namespace, &service.ObjectMeta - availableLabels = service.Labels - } - - if !mutationRequired(ignoredNamespaces, objectMeta) { - glog.Infof("Skipping validation for %s/%s due to policy check", resourceNamespace, resourceName) - return &v1beta1.AdmissionResponse{ - Allowed: true, - } - } - - annotations := map[string]string{admissionWebhookAnnotationStatusKey: "mutated"} - patchBytes, err := createPatch(availableAnnotations, annotations, availableLabels, addLabels) - if err != nil { - return &v1beta1.AdmissionResponse{ - Result: &metav1.Status{ - Message: err.Error(), - }, - } - } - - glog.Infof("AdmissionResponse: patch=%v\n", string(patchBytes)) - return &v1beta1.AdmissionResponse{ - Allowed: true, - Patch: patchBytes, - PatchType: func() *v1beta1.PatchType { - pt := v1beta1.PatchTypeJSONPatch - return &pt - }(), - } -} - -func mutationRequired(ignoredList []string, metadata *metav1.ObjectMeta) bool { - required := admissionRequired(ignoredList, admissionWebhookAnnotationMutateKey, metadata) - annotations := metadata.GetAnnotations() - - if annotations == nil { - annotations = map[string]string{} - } - - status := annotations[admissionWebhookAnnotationStatusKey] - if strings.ToLower(status) == "mutated" { - required = false - } - - glog.Infof("Mutation policy for %v/%v: required:%v", metadata.Namespace, metadata.Name, required) - return required -} - -func createPatch(availableAnnotations map[string]string, annotations map[string]string, availableLabels map[string]string, labels map[string]string) ([]byte, error) { - var patch []patchOperation - patch = append(patch, updateAnnotation(availableAnnotations, annotations)...) - patch = append(patch, updateLabels(availableLabels, labels)...) - return json.Marshal(patch) -} - -func updateAnnotation(target map[string]string, added map[string]string) (patch []patchOperation) { - for key, value := range added { - if target == nil || target[key] == "" { - target = map[string]string{} - patch = append(patch, patchOperation{ - Op: "add", - Path: "/metadata/annotations", - Value: map[string]string{ - key: value, - }, - }) - } else { - patch = append(patch, patchOperation{ - Op: "replace", - Path: "/metadata/annotations/" + key, - Value: value, - }) - } - } - return patch -} - -func updateLabels(target map[string]string, added map[string]string) (patch []patchOperation) { - values := make(map[string]string) - for key, value := range added { - if target == nil || target[key] == "" { - values[key] = value - } - } - - patch = append(patch, patchOperation{ - Op: "add", - Path: "/metadata/labels", - Value: values, - }) - - return patch -} diff --git a/pkg/webhook/server.go b/pkg/webhook/server.go deleted file mode 100644 index b69d658c..00000000 --- a/pkg/webhook/server.go +++ /dev/null @@ -1,220 +0,0 @@ -/* -Copyright 2018 The kube-fledged authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package webhook - -import ( - "crypto/tls" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "strings" - - "github.com/golang/glog" - "k8s.io/api/admission/v1beta1" - admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer" -) - -var ( - runtimeScheme = runtime.NewScheme() - codecs = serializer.NewCodecFactory(runtimeScheme) - deserializer = codecs.UniversalDeserializer() - // (https://github.com/kubernetes/kubernetes/issues/57982) - defaulter = runtime.ObjectDefaulter(runtimeScheme) -) - -var ( - ignoredNamespaces = []string{ - metav1.NamespaceSystem, - metav1.NamespacePublic, - } - - requiredLabels = []string{ - nameLabel, - instanceLabel, - versionLabel, - componentLabel, - partOfLabel, - managedByLabel, - } - - addLabels = map[string]string{ - nameLabel: NA, - instanceLabel: NA, - versionLabel: NA, - componentLabel: NA, - partOfLabel: NA, - managedByLabel: NA, - } -) - -const ( - admissionWebhookAnnotationValidateKey = "admission-webhook-example.banzaicloud.com/validate" - admissionWebhookAnnotationMutateKey = "admission-webhook-example.banzaicloud.com/mutate" - admissionWebhookAnnotationStatusKey = "admission-webhook-example.banzaicloud.com/status" - nameLabel = "app.kubernetes.io/name" - instanceLabel = "app.kubernetes.io/instance" - versionLabel = "app.kubernetes.io/version" - componentLabel = "app.kubernetes.io/component" - partOfLabel = "app.kubernetes.io/part-of" - managedByLabel = "app.kubernetes.io/managed-by" - // NA denotes Not applicable - NA = "not_available" -) - -// Server denotes a webhook https server listening to validating and mutating webhooks from api-server -type Server struct { - server *http.Server -} - -type patchOperation struct { - Op string `json:"op"` - Path string `json:"path"` - Value interface{} `json:"value,omitempty"` -} - -func init() { - _ = corev1.AddToScheme(runtimeScheme) - _ = admissionregistrationv1beta1.AddToScheme(runtimeScheme) - // defaulting with webhooks: - // https://github.com/kubernetes/kubernetes/issues/57982 - // _ = v1.AddToScheme(runtimeScheme) -} - -// CreateAndStartWebHookServer creates and starts and https webhook server -func CreateAndStartWebHookServer(stopCh <-chan struct{}, port int) error { - certFile, keyFile := "", "" - pair, err := tls.LoadX509KeyPair(certFile, keyFile) - if err != nil { - glog.Errorf("Failed to load key pair: %v", err) - return err - } - - whsvr := &Server{ - server: &http.Server{ - Addr: fmt.Sprintf(":%v", port), - TLSConfig: &tls.Config{Certificates: []tls.Certificate{pair}}, - }, - } - - // define http server and server handler - mux := http.NewServeMux() - mux.HandleFunc("/mutate", whsvr.serve) - mux.HandleFunc("/validate", whsvr.serve) - whsvr.server.Handler = mux - - if err := whsvr.server.ListenAndServeTLS("", ""); err != nil { - glog.Errorf("Failed to listen and serve webhook server: %v", err) - return err - } - - <-stopCh - glog.Info("Shutting down webhook server") - return nil -} - -// Serve method for webhook server -func (whsvr *Server) serve(w http.ResponseWriter, r *http.Request) { - var body []byte - if r.Body != nil { - if data, err := ioutil.ReadAll(r.Body); err == nil { - body = data - } - } - - if len(body) == 0 { - glog.Error("empty body") - http.Error(w, "empty body", http.StatusBadRequest) - return - } - - // verify the content type is accurate - contentType := r.Header.Get("Content-Type") - if contentType != "application/json" { - glog.Errorf("Content-Type=%s, expect application/json", contentType) - http.Error(w, "invalid Content-Type, expect `application/json`", http.StatusUnsupportedMediaType) - return - } - - var admissionResponse *v1beta1.AdmissionResponse - ar := v1beta1.AdmissionReview{} - if _, _, err := deserializer.Decode(body, nil, &ar); err != nil { - glog.Errorf("Can't decode body: %v", err) - admissionResponse = &v1beta1.AdmissionResponse{ - Result: &metav1.Status{ - Message: err.Error(), - }, - } - } else { - fmt.Println(r.URL.Path) - if r.URL.Path == "/mutate" { - admissionResponse = whsvr.mutate(&ar) - } else if r.URL.Path == "/validate" { - admissionResponse = whsvr.validate(&ar) - } - } - - admissionReview := v1beta1.AdmissionReview{} - if admissionResponse != nil { - admissionReview.Response = admissionResponse - if ar.Request != nil { - admissionReview.Response.UID = ar.Request.UID - } - } - - resp, err := json.Marshal(admissionReview) - if err != nil { - glog.Errorf("Can't encode response: %v", err) - http.Error(w, fmt.Sprintf("could not encode response: %v", err), http.StatusInternalServerError) - } - - glog.Infof("Ready to write reponse ...") - if _, err := w.Write(resp); err != nil { - glog.Errorf("Can't write response: %v", err) - http.Error(w, fmt.Sprintf("could not write response: %v", err), http.StatusInternalServerError) - } - -} - -func admissionRequired(ignoredList []string, admissionAnnotationKey string, metadata *metav1.ObjectMeta) bool { - // skip special kubernetes system namespaces - for _, namespace := range ignoredList { - if metadata.Namespace == namespace { - glog.Infof("Skip validation for %v for it's in special namespace:%v", metadata.Name, metadata.Namespace) - return false - } - } - - annotations := metadata.GetAnnotations() - if annotations == nil { - annotations = map[string]string{} - } - - var required bool - switch strings.ToLower(annotations[admissionAnnotationKey]) { - default: - required = true - case "n", "no", "false", "off": - required = false - } - - return required -} diff --git a/pkg/webhook/validate.go b/pkg/webhook/validate.go deleted file mode 100644 index ef8c42bb..00000000 --- a/pkg/webhook/validate.go +++ /dev/null @@ -1,102 +0,0 @@ -/* -Copyright 2018 The kube-fledged authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package webhook - -import ( - "encoding/json" - - "github.com/golang/glog" - "k8s.io/api/admission/v1beta1" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func (whsvr *Server) validate(ar *v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { - req := ar.Request - var ( - availableLabels map[string]string - objectMeta *metav1.ObjectMeta - resourceNamespace, resourceName string - ) - - glog.Infof("AdmissionReview for Kind=%v, Namespace=%v Name=%v (%v) UID=%v patchOperation=%v UserInfo=%v", - req.Kind, req.Namespace, req.Name, resourceName, req.UID, req.Operation, req.UserInfo) - - switch req.Kind.Kind { - case "Deployment": - var deployment appsv1.Deployment - if err := json.Unmarshal(req.Object.Raw, &deployment); err != nil { - glog.Errorf("Could not unmarshal raw object: %v", err) - return &v1beta1.AdmissionResponse{ - Result: &metav1.Status{ - Message: err.Error(), - }, - } - } - - resourceName, resourceNamespace, objectMeta = deployment.Name, deployment.Namespace, &deployment.ObjectMeta - availableLabels = deployment.Labels - - case "Service": - var service corev1.Service - if err := json.Unmarshal(req.Object.Raw, &service); err != nil { - glog.Errorf("Could not unmarshal raw object: %v", err) - return &v1beta1.AdmissionResponse{ - Result: &metav1.Status{ - Message: err.Error(), - }, - } - } - - resourceName, resourceNamespace, objectMeta = service.Name, service.Namespace, &service.ObjectMeta - availableLabels = service.Labels - } - - if !validationRequired(ignoredNamespaces, objectMeta) { - glog.Infof("Skipping validation for %s/%s due to policy check", resourceNamespace, resourceName) - return &v1beta1.AdmissionResponse{ - Allowed: true, - } - } - - allowed := true - var result *metav1.Status - glog.Info("available labels:", availableLabels) - glog.Info("required labels", requiredLabels) - - for _, rl := range requiredLabels { - if _, ok := availableLabels[rl]; !ok { - allowed = false - result = &metav1.Status{ - Reason: "required labels are not set", - } - break - } - } - - return &v1beta1.AdmissionResponse{ - Allowed: allowed, - Result: result, - } -} - -func validationRequired(ignoredList []string, metadata *metav1.ObjectMeta) bool { - required := admissionRequired(ignoredList, admissionWebhookAnnotationValidateKey, metadata) - glog.Infof("Validation policy for %v/%v: required:%v", metadata.Namespace, metadata.Name, required) - return required -} From 5da6814ecea56bbc6bcf0d82a21f347fa5b57a90 Mon Sep 17 00:00:00 2001 From: bhuvanessr Date: Wed, 27 May 2020 19:29:01 +0530 Subject: [PATCH 06/21] added code for webhook server --- Makefile | 416 +++++++++--------- build/Dockerfile.webhook_server | 30 ++ ...ledged-deployment-fledged-controller.yaml} | 0 ...kubefledged-deployment-webhook-server.yaml | 38 ++ deploy/kubefledged-validatingwebhook.yaml | 2 +- 5 files changed, 284 insertions(+), 202 deletions(-) create mode 100644 build/Dockerfile.webhook_server rename deploy/{kubefledged-deployment.yaml => kubefledged-deployment-fledged-controller.yaml} (100%) create mode 100644 deploy/kubefledged-deployment-webhook-server.yaml diff --git a/Makefile b/Makefile index 8576c1c4..749b0896 100644 --- a/Makefile +++ b/Makefile @@ -1,201 +1,215 @@ -# Copyright 2018 The kube-fledged authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -.PHONY: clean clean-fledged clean-client clean-operator fledged-amd64 fledged-image client-image operator-image build-images push-images test deploy update remove -# Default tag and architecture. Can be overridden -TAG?=$(shell git describe --tags --dirty) -ARCH?=amd64 -# Only enable CGO (and build the UDP backend) on AMD64 -ifeq ($(ARCH),amd64) - CGO_ENABLED=1 -else - CGO_ENABLED=0 -endif - -GOARM=7 - -ifndef FLEDGED_IMAGE_REPO - FLEDGED_IMAGE_REPO=docker.io/senthilrch/fledged -endif - -ifndef FLEDGED_DOCKER_CLIENT_IMAGE_REPO - FLEDGED_DOCKER_CLIENT_IMAGE_REPO=docker.io/senthilrch/fledged-docker-client -endif - -ifndef OPERATOR_IMAGE_REPO - OPERATOR_IMAGE_REPO=docker.io/senthilrch/kubefledged-operator -endif - -ifndef RELEASE_VERSION - RELEASE_VERSION=v0.7.0 -endif - -ifndef DOCKER_VERSION - DOCKER_VERSION=19.03.8 -endif - -ifndef CRICTL_VERSION - CRICTL_VERSION=v1.18.0 -endif - -ifndef GOLANG_VERSION - GOLANG_VERSION=1.14.2 -endif - -ifndef ALPINE_VERSION - ALPINE_VERSION=3.11.6 -endif - -ifndef OPERATORSDK_VERSION - OPERATORSDK_VERSION=v0.17.0 -endif - -ifndef GIT_BRANCH - GIT_BRANCH=master -endif - -ifndef TARGET_PLATFORMS - TARGET_PLATFORMS=linux/amd64,linux/arm/v7,linux/arm64/v8 -endif - -ifndef OPERATOR_TARGET_PLATFORMS - OPERATOR_TARGET_PLATFORMS=linux/amd64,linux/arm64 -endif - -ifndef BUILD_OUTPUT - BUILD_OUTPUT=--push -endif - -HTTP_PROXY_CONFIG= -ifdef HTTP_PROXY - HTTP_PROXY_CONFIG=--build-arg http_proxy=${HTTP_PROXY} -endif - -HTTPS_PROXY_CONFIG= -ifdef HTTPS_PROXY - HTTPS_PROXY_CONFIG=--build-arg https_proxy=${HTTPS_PROXY} -endif - - -### BUILD -clean: clean-fledged clean-client clean-operator - -clean-fledged: - -rm -f build/fledged - -docker image rm ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION} - -docker image rm `docker image ls -f dangling=true -q` - -clean-client: - -docker image rm ${FLEDGED_DOCKER_CLIENT_IMAGE_REPO}:${RELEASE_VERSION} - -docker image rm `docker image ls -f dangling=true -q` - -clean-operator: - -docker image rm ${OPERATOR_IMAGE_REPO}:${RELEASE_VERSION} - -docker image rm `docker image ls -f dangling=true -q` - -fledged-amd64: clean-fledged - CGO_ENABLED=0 go build -o build/fledged -ldflags '-s -w -extldflags "-static"' cmd/fledged.go && \ - cd build && docker build -t ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION} -f Dockerfile.fledged_dev \ - --build-arg ALPINE_VERSION=${ALPINE_VERSION} . - -fledged-dev: fledged-amd64 - docker push ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION} - -fledged-image: clean-fledged - cd build && docker buildx build --platform=${TARGET_PLATFORMS} -t ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION} \ - -f Dockerfile.fledged ${HTTP_PROXY_CONFIG} ${HTTPS_PROXY_CONFIG} --build-arg GIT_BRANCH=${GIT_BRANCH} \ - --build-arg GOLANG_VERSION=${GOLANG_VERSION} --build-arg ALPINE_VERSION=${ALPINE_VERSION} --progress=plain ${BUILD_OUTPUT} . - -client-image: clean-client - cd build && docker buildx build --platform=${TARGET_PLATFORMS} -t ${FLEDGED_DOCKER_CLIENT_IMAGE_REPO}:${RELEASE_VERSION} \ - -f Dockerfile.docker_client ${HTTP_PROXY_CONFIG} ${HTTPS_PROXY_CONFIG} \ - --build-arg DOCKER_VERSION=${DOCKER_VERSION} --build-arg CRICTL_VERSION=${CRICTL_VERSION} \ - --build-arg ALPINE_VERSION=${ALPINE_VERSION} --progress=plain ${BUILD_OUTPUT} . - -operator-image: clean-operator - cd deploy/kubefledged-operator && \ - docker buildx build --platform=${OPERATOR_TARGET_PLATFORMS} -t ${OPERATOR_IMAGE_REPO}:${RELEASE_VERSION} \ - -f ./build/Dockerfile --build-arg OPERATORSDK_VERSION=${OPERATORSDK_VERSION} --progress=plain ${BUILD_OUTPUT} . - -push-images: - -docker push ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION} - -docker push ${FLEDGED_DOCKER_CLIENT_IMAGE_REPO}:${RELEASE_VERSION} - -docker push ${OPERATOR_IMAGE_REPO}:${RELEASE_VERSION} - -release: install-buildx fledged-image client-image operator-image - -install-buildx: - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - -docker buildx rm multibuilder - docker buildx create --name multibuilder --driver docker-container --use - docker buildx inspect --bootstrap - docker buildx ls - -test: - -rm -f coverage.out - bash hack/run-unit-tests.sh - -deploy-using-yaml: - kubectl apply -f deploy/kubefledged-crd.yaml && \ - kubectl apply -f deploy/kubefledged-namespace.yaml && \ - kubectl apply -f deploy/kubefledged-serviceaccount.yaml && \ - kubectl apply -f deploy/kubefledged-clusterrole.yaml && \ - kubectl apply -f deploy/kubefledged-clusterrolebinding.yaml && \ - kubectl apply -f deploy/kubefledged-deployment.yaml - -deploy-using-operator: - # Deploy the operator to a separate namespace called "operators" - sed -i "s|OPERATOR_NAMESPACE|operators|g" deploy/kubefledged-operator/deploy/service_account.yaml - sed -i "s|OPERATOR_NAMESPACE|operators|g" deploy/kubefledged-operator/deploy/clusterrole_binding.yaml - sed -i "s|OPERATOR_NAMESPACE|operators|g" deploy/kubefledged-operator/deploy/operator.yaml - -kubectl create namespace operators - kubectl create -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml - kubectl create -f deploy/kubefledged-operator/deploy/service_account.yaml - kubectl create -f deploy/kubefledged-operator/deploy/clusterrole.yaml - kubectl create -f deploy/kubefledged-operator/deploy/clusterrole_binding.yaml - kubectl create -f deploy/kubefledged-operator/deploy/operator.yaml - # Deploy kube-fledged to a separate namespace called "kube-fledged" - sed -i "s|OPERATOR_NAMESPACE|operators|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml - sed -i "s|KUBEFLEDGED_NAMESPACE|kube-fledged|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml - -kubectl create namespace kube-fledged - kubectl create -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml - -update: - kubectl scale deployment kubefledged --replicas=0 -n kube-fledged && sleep 1 && \ - kubectl scale deployment kubefledged --replicas=1 -n kube-fledged && sleep 1 && \ - kubectl get pods -l app=kubefledged -n kube-fledged - -remove: - kubectl delete -f deploy/kubefledged-namespace.yaml && \ - kubectl delete -f deploy/kubefledged-clusterrolebinding.yaml && \ - kubectl delete -f deploy/kubefledged-clusterrole.yaml && \ - kubectl delete -f deploy/kubefledged-crd.yaml - -remove-all: - # Remove kube-fledged and the namespace "kube-fledged" - kubectl delete -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml - -kubectl delete namespace kube-fledged - sed -i "s|kube-fledged|KUBEFLEDGED_NAMESPACE|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml - sed -i "s|operators|OPERATOR_NAMESPACE|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml - # Remove the operator and the namespace "operators" - kubectl delete -f deploy/kubefledged-operator/deploy/operator.yaml - kubectl delete -f deploy/kubefledged-operator/deploy/clusterrole_binding.yaml - kubectl delete -f deploy/kubefledged-operator/deploy/clusterrole.yaml - kubectl delete -f deploy/kubefledged-operator/deploy/service_account.yaml - kubectl delete -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml - -kubectl delete namespace operators - sed -i "s|operators|OPERATOR_NAMESPACE|g" deploy/kubefledged-operator/deploy/operator.yaml - sed -i "s|operators|OPERATOR_NAMESPACE|g" deploy/kubefledged-operator/deploy/clusterrole_binding.yaml - sed -i "s|operators|OPERATOR_NAMESPACE|g" deploy/kubefledged-operator/deploy/service_account.yaml - +# Copyright 2018 The kube-fledged authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.PHONY: clean clean-fledged clean-client clean-operator fledged-amd64 fledged-image client-image operator-image build-images push-images test deploy update remove +# Default tag and architecture. Can be overridden +TAG?=$(shell git describe --tags --dirty) +ARCH?=amd64 +# Only enable CGO (and build the UDP backend) on AMD64 +ifeq ($(ARCH),amd64) + CGO_ENABLED=1 +else + CGO_ENABLED=0 +endif + +GOARM=7 + +ifndef FLEDGED_IMAGE_REPO + FLEDGED_IMAGE_REPO=docker.io/senthilrch/fledged +endif + +ifndef WEBHOOK_SERVER_IMAGE_REPO + WEBHOOK_SERVER_IMAGE_REPO=docker.io/senthilrch/fledged-webhook-server +endif + +ifndef FLEDGED_DOCKER_CLIENT_IMAGE_REPO + FLEDGED_DOCKER_CLIENT_IMAGE_REPO=docker.io/senthilrch/fledged-docker-client +endif + +ifndef OPERATOR_IMAGE_REPO + OPERATOR_IMAGE_REPO=docker.io/senthilrch/kubefledged-operator +endif + +ifndef RELEASE_VERSION + RELEASE_VERSION=v0.7.0 +endif + +ifndef DOCKER_VERSION + DOCKER_VERSION=19.03.8 +endif + +ifndef CRICTL_VERSION + CRICTL_VERSION=v1.18.0 +endif + +ifndef GOLANG_VERSION + GOLANG_VERSION=1.14.2 +endif + +ifndef ALPINE_VERSION + ALPINE_VERSION=3.11.6 +endif + +ifndef OPERATORSDK_VERSION + OPERATORSDK_VERSION=v0.17.0 +endif + +ifndef GIT_BRANCH + GIT_BRANCH=master +endif + +ifndef TARGET_PLATFORMS + TARGET_PLATFORMS=linux/amd64,linux/arm/v7,linux/arm64/v8 +endif + +ifndef OPERATOR_TARGET_PLATFORMS + OPERATOR_TARGET_PLATFORMS=linux/amd64,linux/arm64 +endif + +ifndef BUILD_OUTPUT + BUILD_OUTPUT=--push +endif + +HTTP_PROXY_CONFIG= +ifdef HTTP_PROXY + HTTP_PROXY_CONFIG=--build-arg http_proxy=${HTTP_PROXY} +endif + +HTTPS_PROXY_CONFIG= +ifdef HTTPS_PROXY + HTTPS_PROXY_CONFIG=--build-arg https_proxy=${HTTPS_PROXY} +endif + + +### BUILD +clean: clean-fledged clean-webhook-server clean-client clean-operator + +clean-fledged: + -rm -f build/fledged + -docker image rm ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION} + -docker image rm `docker image ls -f dangling=true -q` + +clean-webhook-server: + -rm -f build/webhook-server + -docker image rm ${WEBHOOK_SERVER_IMAGE_REPO}:${RELEASE_VERSION} + -docker image rm `docker image ls -f dangling=true -q` + +clean-client: + -docker image rm ${FLEDGED_DOCKER_CLIENT_IMAGE_REPO}:${RELEASE_VERSION} + -docker image rm `docker image ls -f dangling=true -q` + +clean-operator: + -docker image rm ${OPERATOR_IMAGE_REPO}:${RELEASE_VERSION} + -docker image rm `docker image ls -f dangling=true -q` + +fledged-amd64: clean-fledged + CGO_ENABLED=0 go build -o build/fledged -ldflags '-s -w -extldflags "-static"' cmd/fledged.go && \ + cd build && docker build -t ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION} -f Dockerfile.fledged_dev \ + --build-arg ALPINE_VERSION=${ALPINE_VERSION} . + +fledged-dev: fledged-amd64 + docker push ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION} + +fledged-image: clean-fledged + cd build && docker buildx build --platform=${TARGET_PLATFORMS} -t ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION} \ + -f Dockerfile.fledged ${HTTP_PROXY_CONFIG} ${HTTPS_PROXY_CONFIG} --build-arg GIT_BRANCH=${GIT_BRANCH} \ + --build-arg GOLANG_VERSION=${GOLANG_VERSION} --build-arg ALPINE_VERSION=${ALPINE_VERSION} --progress=plain ${BUILD_OUTPUT} . + +webhook-server-image: clean-webhook-server + cd build && docker buildx build --platform=${TARGET_PLATFORMS} -t ${WEBHOOK_SERVER_IMAGE_REPO}:${RELEASE_VERSION} \ + -f Dockerfile.webhook_server ${HTTP_PROXY_CONFIG} ${HTTPS_PROXY_CONFIG} --build-arg GIT_BRANCH=${GIT_BRANCH} \ + --build-arg GOLANG_VERSION=${GOLANG_VERSION} --build-arg ALPINE_VERSION=${ALPINE_VERSION} --progress=plain ${BUILD_OUTPUT} . + +client-image: clean-client + cd build && docker buildx build --platform=${TARGET_PLATFORMS} -t ${FLEDGED_DOCKER_CLIENT_IMAGE_REPO}:${RELEASE_VERSION} \ + -f Dockerfile.docker_client ${HTTP_PROXY_CONFIG} ${HTTPS_PROXY_CONFIG} \ + --build-arg DOCKER_VERSION=${DOCKER_VERSION} --build-arg CRICTL_VERSION=${CRICTL_VERSION} \ + --build-arg ALPINE_VERSION=${ALPINE_VERSION} --progress=plain ${BUILD_OUTPUT} . + +operator-image: clean-operator + cd deploy/kubefledged-operator && \ + docker buildx build --platform=${OPERATOR_TARGET_PLATFORMS} -t ${OPERATOR_IMAGE_REPO}:${RELEASE_VERSION} \ + -f ./build/Dockerfile --build-arg OPERATORSDK_VERSION=${OPERATORSDK_VERSION} --progress=plain ${BUILD_OUTPUT} . + +push-images: + -docker push ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION} + -docker push ${FLEDGED_DOCKER_CLIENT_IMAGE_REPO}:${RELEASE_VERSION} + -docker push ${OPERATOR_IMAGE_REPO}:${RELEASE_VERSION} + +release: install-buildx fledged-image client-image operator-image + +install-buildx: + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + -docker buildx rm multibuilder + docker buildx create --name multibuilder --driver docker-container --use + docker buildx inspect --bootstrap + docker buildx ls + +test: + -rm -f coverage.out + bash hack/run-unit-tests.sh + +deploy-using-yaml: + kubectl apply -f deploy/kubefledged-crd.yaml && \ + kubectl apply -f deploy/kubefledged-namespace.yaml && \ + kubectl apply -f deploy/kubefledged-serviceaccount.yaml && \ + kubectl apply -f deploy/kubefledged-clusterrole.yaml && \ + kubectl apply -f deploy/kubefledged-clusterrolebinding.yaml && \ + kubectl apply -f deploy/kubefledged-deployment.yaml + +deploy-using-operator: + # Deploy the operator to a separate namespace called "operators" + sed -i "s|OPERATOR_NAMESPACE|operators|g" deploy/kubefledged-operator/deploy/service_account.yaml + sed -i "s|OPERATOR_NAMESPACE|operators|g" deploy/kubefledged-operator/deploy/clusterrole_binding.yaml + sed -i "s|OPERATOR_NAMESPACE|operators|g" deploy/kubefledged-operator/deploy/operator.yaml + -kubectl create namespace operators + kubectl create -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml + kubectl create -f deploy/kubefledged-operator/deploy/service_account.yaml + kubectl create -f deploy/kubefledged-operator/deploy/clusterrole.yaml + kubectl create -f deploy/kubefledged-operator/deploy/clusterrole_binding.yaml + kubectl create -f deploy/kubefledged-operator/deploy/operator.yaml + # Deploy kube-fledged to a separate namespace called "kube-fledged" + sed -i "s|OPERATOR_NAMESPACE|operators|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml + sed -i "s|KUBEFLEDGED_NAMESPACE|kube-fledged|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml + -kubectl create namespace kube-fledged + kubectl create -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml + +update: + kubectl scale deployment kubefledged --replicas=0 -n kube-fledged && sleep 1 && \ + kubectl scale deployment kubefledged --replicas=1 -n kube-fledged && sleep 1 && \ + kubectl get pods -l app=kubefledged -n kube-fledged + +remove: + kubectl delete -f deploy/kubefledged-namespace.yaml && \ + kubectl delete -f deploy/kubefledged-clusterrolebinding.yaml && \ + kubectl delete -f deploy/kubefledged-clusterrole.yaml && \ + kubectl delete -f deploy/kubefledged-crd.yaml + +remove-all: + # Remove kube-fledged and the namespace "kube-fledged" + kubectl delete -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml + -kubectl delete namespace kube-fledged + sed -i "s|kube-fledged|KUBEFLEDGED_NAMESPACE|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml + sed -i "s|operators|OPERATOR_NAMESPACE|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml + # Remove the operator and the namespace "operators" + kubectl delete -f deploy/kubefledged-operator/deploy/operator.yaml + kubectl delete -f deploy/kubefledged-operator/deploy/clusterrole_binding.yaml + kubectl delete -f deploy/kubefledged-operator/deploy/clusterrole.yaml + kubectl delete -f deploy/kubefledged-operator/deploy/service_account.yaml + kubectl delete -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml + -kubectl delete namespace operators + sed -i "s|operators|OPERATOR_NAMESPACE|g" deploy/kubefledged-operator/deploy/operator.yaml + sed -i "s|operators|OPERATOR_NAMESPACE|g" deploy/kubefledged-operator/deploy/clusterrole_binding.yaml + sed -i "s|operators|OPERATOR_NAMESPACE|g" deploy/kubefledged-operator/deploy/service_account.yaml + diff --git a/build/Dockerfile.webhook_server b/build/Dockerfile.webhook_server new file mode 100644 index 00000000..0446fffc --- /dev/null +++ b/build/Dockerfile.webhook_server @@ -0,0 +1,30 @@ +# Copyright 2018 The kube-fledged authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG GOLANG_VERSION +ARG ALPINE_VERSION + +FROM golang:$GOLANG_VERSION AS builder +LABEL stage=builder +ARG GIT_BRANCH +RUN mkdir -p /go/src/github.com/senthilrch && \ + git clone --depth=1 --single-branch --branch=$GIT_BRANCH https://github.com/senthilrch/kube-fledged /go/src/github.com/senthilrch/kube-fledged && \ + cd /go/src/github.com/senthilrch/kube-fledged && \ + CGO_ENABLED=0 go build -o build/fledged-webhook-server -ldflags '-s -w -extldflags "-static"' cmd/webhook-server/main.go + +FROM alpine:$ALPINE_VERSION +LABEL maintainer="senthilrch " +COPY --from=builder /go/src/github.com/senthilrch/kube-fledged/build/fledged-webhook-server /opt/bin/fledged-webhook-server +RUN chmod 755 /opt/bin/fledged-webhook-server +ENTRYPOINT ["/opt/bin/fledged-webhook-server"] diff --git a/deploy/kubefledged-deployment.yaml b/deploy/kubefledged-deployment-fledged-controller.yaml similarity index 100% rename from deploy/kubefledged-deployment.yaml rename to deploy/kubefledged-deployment-fledged-controller.yaml diff --git a/deploy/kubefledged-deployment-webhook-server.yaml b/deploy/kubefledged-deployment-webhook-server.yaml new file mode 100644 index 00000000..a3966807 --- /dev/null +++ b/deploy/kubefledged-deployment-webhook-server.yaml @@ -0,0 +1,38 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kubefledged-webhook-server + namespace: kube-fledged +spec: + replicas: 1 + selector: + matchLabels: + app: kubefledged + template: + metadata: + labels: + app: kubefledged + spec: + containers: + - image: senthilrch/fledged-webhook-server:v0.7.0 + command: ["/opt/bin/fledged-webhook-server"] + args: + - "--stderrthreshold=INFO" + - "--cert-file=/var/run/secrets/webhook-server/cert.pem" + - "--key-file=/var/run/secrets/webhook-server/key.pem" + - "--port=443" + imagePullPolicy: Always + name: webhook-server + env: + - name: KUBEFLEDGED_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: secret-volume + mountPath: "/var/run/secrets/webhook-server" + readOnly: true + volumes: + - name: secret-volume + secret: + secretName: kubefledged-webhook-server diff --git a/deploy/kubefledged-validatingwebhook.yaml b/deploy/kubefledged-validatingwebhook.yaml index bb2bee2d..4c018193 100644 --- a/deploy/kubefledged-validatingwebhook.yaml +++ b/deploy/kubefledged-validatingwebhook.yaml @@ -10,7 +10,7 @@ webhooks: clientConfig: service: namespace: kube-fledged - name: kubefledged-webhook-svc + name: kubefledged-webhook-server path: "/validate-image-cache" port: 443 rules: From 3431300f3414b3317e9bdc2f2795f2e441387284 Mon Sep 17 00:00:00 2001 From: bhuvanessr Date: Wed, 27 May 2020 19:33:30 +0530 Subject: [PATCH 07/21] enable docker experimental in makefile --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 749b0896..f9f88c97 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,7 @@ else endif GOARM=7 +DOCKER_CLI_EXPERIMENTAL=enabled ifndef FLEDGED_IMAGE_REPO FLEDGED_IMAGE_REPO=docker.io/senthilrch/fledged From 55e70b7ba6db9d2d7ef2921db435136d1f002cd0 Mon Sep 17 00:00:00 2001 From: bhuvanessr Date: Wed, 27 May 2020 19:56:57 +0530 Subject: [PATCH 08/21] made changes to makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f9f88c97..7e85d067 100644 --- a/Makefile +++ b/Makefile @@ -115,7 +115,7 @@ clean-operator: -docker image rm `docker image ls -f dangling=true -q` fledged-amd64: clean-fledged - CGO_ENABLED=0 go build -o build/fledged -ldflags '-s -w -extldflags "-static"' cmd/fledged.go && \ + CGO_ENABLED=0 go build -o build/fledged -ldflags '-s -w -extldflags "-static"' cmd/fledged/main.go && \ cd build && docker build -t ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION} -f Dockerfile.fledged_dev \ --build-arg ALPINE_VERSION=${ALPINE_VERSION} . From acac5f08e816bb781564e478bf453004b6853a39 Mon Sep 17 00:00:00 2001 From: bhuvanessr Date: Wed, 27 May 2020 19:59:00 +0530 Subject: [PATCH 09/21] fix gofmt errors --- cmd/fledged/app/controller.go | 1486 ++++++------- cmd/fledged/app/controller_helpers.go | 146 +- cmd/fledged/app/controller_helpers_test.go | 384 ++-- cmd/fledged/app/controller_test.go | 2350 ++++++++++---------- 4 files changed, 2183 insertions(+), 2183 deletions(-) diff --git a/cmd/fledged/app/controller.go b/cmd/fledged/app/controller.go index 74fdf6df..0299eeda 100644 --- a/cmd/fledged/app/controller.go +++ b/cmd/fledged/app/controller.go @@ -1,743 +1,743 @@ -/* -Copyright 2018 The kube-fledged authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "fmt" - "reflect" - "time" - - "github.com/golang/glog" - fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" - clientset "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned" - fledgedscheme "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/scheme" - informers "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions/fledged/v1alpha1" - listers "github.com/senthilrch/kube-fledged/pkg/client/listers/fledged/v1alpha1" - "github.com/senthilrch/kube-fledged/pkg/images" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/wait" - coreinformers "k8s.io/client-go/informers/core/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/scheme" - typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" - corelisters "k8s.io/client-go/listers/core/v1" - "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/record" - "k8s.io/client-go/util/workqueue" -) - -const controllerAgentName = "fledged" -const fledgedCacheSpecValidationKey = "fledged.k8s.io/cachespecvalidation" -const imageCachePurgeAnnotationKey = "fledged.k8s.io/purge-imagecache" -const imageCacheRefreshAnnotationKey = "fledged.k8s.io/refresh-imagecache" - -const ( - // SuccessSynced is used as part of the Event 'reason' when a ImageCache is synced - SuccessSynced = "Synced" - // MessageResourceSynced is the message used for an Event fired when a ImageCache - // is synced successfully - MessageResourceSynced = "ImageCache synced successfully" -) - -// Controller is the controller for ImageCache resources -type Controller struct { - // kubeclientset is a standard kubernetes clientset - kubeclientset kubernetes.Interface - // fledgedclientset is a clientset for fledged.k8s.io API group - fledgedclientset clientset.Interface - - fledgedNameSpace string - nodesLister corelisters.NodeLister - nodesSynced cache.InformerSynced - imageCachesLister listers.ImageCacheLister - imageCachesSynced cache.InformerSynced - - // workqueue is a rate limited work queue. This is used to queue work to be - // processed instead of performing it as soon as a change happens. This - // means we can ensure we only process a fixed amount of resources at a - // time, and makes it easy to ensure we are never processing the same item - // simultaneously in two different workers. - workqueue workqueue.RateLimitingInterface - imageworkqueue workqueue.RateLimitingInterface - imageManager *images.ImageManager - // recorder is an event recorder for recording Event resources to the - // Kubernetes API. - recorder record.EventRecorder - imageCacheRefreshFrequency time.Duration -} - -// NewController returns a new fledged controller -func NewController( - kubeclientset kubernetes.Interface, - fledgedclientset clientset.Interface, - namespace string, - nodeInformer coreinformers.NodeInformer, - imageCacheInformer informers.ImageCacheInformer, - imageCacheRefreshFrequency time.Duration, - imagePullDeadlineDuration time.Duration, - dockerClientImage string, - imagePullPolicy string) *Controller { - - utilruntime.Must(fledgedscheme.AddToScheme(scheme.Scheme)) - glog.V(4).Info("Creating event broadcaster") - eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) - eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: kubeclientset.CoreV1().Events("")}) - recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerAgentName}) - - controller := &Controller{ - kubeclientset: kubeclientset, - fledgedclientset: fledgedclientset, - fledgedNameSpace: namespace, - nodesLister: nodeInformer.Lister(), - nodesSynced: nodeInformer.Informer().HasSynced, - imageCachesLister: imageCacheInformer.Lister(), - imageCachesSynced: imageCacheInformer.Informer().HasSynced, - workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ImageCaches"), - imageworkqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ImagePullerStatus"), - recorder: recorder, - imageCacheRefreshFrequency: imageCacheRefreshFrequency, - } - - imageManager, _ := images.NewImageManager(controller.workqueue, controller.imageworkqueue, controller.kubeclientset, controller.fledgedNameSpace, imagePullDeadlineDuration, dockerClientImage, imagePullPolicy) - controller.imageManager = imageManager - - glog.Info("Setting up event handlers") - // Set up an event handler for when ImageCache resources change - imageCacheInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - controller.enqueueImageCache(images.ImageCacheCreate, nil, obj) - }, - UpdateFunc: func(old, new interface{}) { - controller.enqueueImageCache(images.ImageCacheUpdate, old, new) - }, - DeleteFunc: func(obj interface{}) { - controller.enqueueImageCache(images.ImageCacheDelete, obj, nil) - }, - }) - return controller -} - -// PreFlightChecks performs pre-flight checks and actions before the controller is started -func (c *Controller) PreFlightChecks() error { - if err := c.danglingJobs(); err != nil { - return err - } - if err := c.danglingImageCaches(); err != nil { - return err - } - return nil -} - -// danglingJobs finds and removes dangling or stuck jobs -func (c *Controller) danglingJobs() error { - joblist, err := c.kubeclientset.BatchV1().Jobs(c.fledgedNameSpace).List(metav1.ListOptions{}) - if err != nil { - glog.Errorf("Error listing jobs: %v", err) - return err - } - - if joblist == nil || len(joblist.Items) == 0 { - glog.Info("No dangling or stuck jobs found...") - return nil - } - deletePropagation := metav1.DeletePropagationBackground - for _, job := range joblist.Items { - err := c.kubeclientset.BatchV1().Jobs(c.fledgedNameSpace). - Delete(job.Name, &metav1.DeleteOptions{PropagationPolicy: &deletePropagation}) - if err != nil { - glog.Errorf("Error deleting job(%s): %v", job.Name, err) - return err - } - glog.Infof("Dangling Job(%s) deleted", job.Name) - } - return nil -} - -// danglingImageCaches finds dangling or stuck image cache and marks them as abhorted. Such -// image caches will get refreshed in the next cycle -func (c *Controller) danglingImageCaches() error { - dangling := false - imagecachelist, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(c.fledgedNameSpace).List(metav1.ListOptions{}) - if err != nil { - glog.Errorf("Error listing imagecaches: %v", err) - return err - } - - if imagecachelist == nil || len(imagecachelist.Items) == 0 { - glog.Info("No dangling or stuck imagecaches found...") - return nil - } - status := &fledgedv1alpha1.ImageCacheStatus{ - Failures: map[string]fledgedv1alpha1.NodeReasonMessageList{}, - Status: fledgedv1alpha1.ImageCacheActionStatusAborted, - Reason: fledgedv1alpha1.ImageCacheReasonImagePullAborted, - Message: fledgedv1alpha1.ImageCacheMessageImagePullAborted, - } - for _, imagecache := range imagecachelist.Items { - if imagecache.Status.Status == fledgedv1alpha1.ImageCacheActionStatusProcessing { - status.StartTime = imagecache.Status.StartTime - err := c.updateImageCacheStatus(&imagecache, status) - if err != nil { - glog.Errorf("Error updating ImageCache(%s) status to '%s': %v", imagecache.Name, fledgedv1alpha1.ImageCacheActionStatusAborted, err) - return err - } - dangling = true - glog.Infof("Dangling Image cache(%s) status changed to '%s'", imagecache.Name, fledgedv1alpha1.ImageCacheActionStatusAborted) - } - } - - if !dangling { - glog.Info("No dangling or stuck imagecaches found...") - } - return nil -} - -// Run will set up the event handlers for types we are interested in, as well -// as syncing informer caches and starting workers. It will block until stopCh -// is closed, at which point it will shutdown the workqueue and wait for -// workers to finish processing their current work items. -func (c *Controller) Run(threadiness int, stopCh <-chan struct{}) error { - defer runtime.HandleCrash() - defer c.workqueue.ShutDown() - defer c.imageworkqueue.ShutDown() - - // Start the informer factories to begin populating the informer caches - glog.Info("Starting fledged controller") - - // Wait for the caches to be synced before starting workers - glog.Info("Waiting for informer caches to sync") - if ok := cache.WaitForCacheSync(stopCh, c.nodesSynced, c.imageCachesSynced); !ok { - return fmt.Errorf("failed to wait for caches to sync") - } - - glog.Info("Starting image cache worker") - // Launch workers to process ImageCache resources - for i := 0; i < threadiness; i++ { - go wait.Until(c.runWorker, time.Second, stopCh) - } - - if c.imageCacheRefreshFrequency.Nanoseconds() != int64(0) { - glog.Info("Starting cache refresh worker") - go wait.Until(c.runRefreshWorker, c.imageCacheRefreshFrequency, stopCh) - } - - glog.Info("Started workers") - c.imageManager.Run(stopCh) - if err := c.imageManager.Run(stopCh); err != nil { - glog.Fatalf("Error running image manager: %s", err.Error()) - } - - <-stopCh - glog.Info("Shutting down workers") - - return nil -} - -// enqueueImageCache takes a ImageCache resource and converts it into a namespace/name -// string which is then put onto the work queue. This method should *not* be -// passed resources of any type other than ImageCache. -func (c *Controller) enqueueImageCache(workType images.WorkType, old, new interface{}) bool { - var key string - var err error - var obj interface{} - wqKey := images.WorkQueueKey{} - - switch workType { - case images.ImageCacheCreate: - obj = new - newImageCache := new.(*fledgedv1alpha1.ImageCache) - // If the ImageCache resource already has a status field, it means it's already - // synced, so do not queue it for processing - if !reflect.DeepEqual(newImageCache.Status, fledgedv1alpha1.ImageCacheStatus{}) { - return false - } - case images.ImageCacheUpdate: - obj = new - oldImageCache := old.(*fledgedv1alpha1.ImageCache) - newImageCache := new.(*fledgedv1alpha1.ImageCache) - - if oldImageCache.Status.Status == fledgedv1alpha1.ImageCacheActionStatusProcessing { - if !reflect.DeepEqual(newImageCache.Spec, oldImageCache.Spec) { - glog.Warningf("Received image cache update/purge/delete for '%s' while it is under processing, so ignoring.", oldImageCache.Name) - return false - } - } - if _, exists := newImageCache.Annotations[imageCachePurgeAnnotationKey]; exists { - if _, exists := oldImageCache.Annotations[imageCachePurgeAnnotationKey]; !exists { - workType = images.ImageCachePurge - break - } - } - if _, exists := newImageCache.Annotations[imageCacheRefreshAnnotationKey]; exists { - if _, exists := oldImageCache.Annotations[imageCacheRefreshAnnotationKey]; !exists { - workType = images.ImageCacheRefresh - break - } - } - if !reflect.DeepEqual(newImageCache.Spec, oldImageCache.Spec) { - if validation, ok := newImageCache.Annotations[fledgedCacheSpecValidationKey]; ok { - if validation == "failed" { - if err := c.removeAnnotation(newImageCache, fledgedCacheSpecValidationKey); err != nil { - glog.Errorf("Error removing Annotation %s from imagecache(%s): %v", fledgedCacheSpecValidationKey, newImageCache.Name, err) - } - return false - } - } - } else { - return false - } - case images.ImageCacheDelete: - return false - - case images.ImageCacheRefresh: - obj = old - } - - if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { - runtime.HandleError(err) - return false - } - wqKey.WorkType = workType - wqKey.ObjKey = key - if workType == images.ImageCacheUpdate { - oldImageCache := old.(*fledgedv1alpha1.ImageCache) - wqKey.OldImageCache = oldImageCache - } - - c.workqueue.AddRateLimited(wqKey) - glog.V(4).Infof("enqueueImageCache::ImageCache resource queued for work type %s", workType) - return true -} - -// runWorker is a long-running function that will continually call the -// processNextWorkItem function in order to read and process a message on the -// workqueue. -func (c *Controller) runWorker() { - for c.processNextWorkItem() { - } -} - -// processNextWorkItem will read a single work item off the workqueue and -// attempt to process it, by calling the syncHandler. -func (c *Controller) processNextWorkItem() bool { - //glog.Info("processNextWorkItem::Beginning...") - obj, shutdown := c.workqueue.Get() - - if shutdown { - return false - } - - // We wrap this block in a func so we can defer c.workqueue.Done. - err := func(obj interface{}) error { - // We call Done here so the workqueue knows we have finished - // processing this item. We also must remember to call Forget if we - // do not want this work item being re-queued. For example, we do - // not call Forget if a transient error occurs, instead the item is - // put back on the workqueue and attempted again after a back-off - // period. - defer c.workqueue.Done(obj) - var key images.WorkQueueKey - var ok bool - // We expect strings to come off the workqueue. These are of the - // form namespace/name. We do this as the delayed nature of the - // workqueue means the items in the informer cache may actually be - // more up to date that when the item was initially put onto the - // workqueue. - if key, ok = obj.(images.WorkQueueKey); !ok { - // As the item in the workqueue is actually invalid, we call - // Forget here else we'd go into a loop of attempting to - // process a work item that is invalid. - c.workqueue.Forget(obj) - runtime.HandleError(fmt.Errorf("Unexpected type in workqueue: %#v", obj)) - return nil - } - // Run the syncHandler, passing it the namespace/name string of the - // ImageCache resource to be synced. - if err := c.syncHandler(key); err != nil { - glog.Errorf("error syncing imagecache: %v", err.Error()) - return fmt.Errorf("error syncing imagecache: %v", err.Error()) - } - // Finally, if no error occurs we Forget this item so it does not - // get queued again until another change happens. - c.workqueue.Forget(obj) - //glog.Infof("Successfully synced '%s' for event '%s'", key.ObjKey, key.WorkType) - return nil - }(obj) - - if err != nil { - runtime.HandleError(err) - return true - } - - return true -} - -// runRefreshWorker is resposible of refreshing the image cache -func (c *Controller) runRefreshWorker() { - // List the ImageCache resources - imageCaches, err := c.imageCachesLister.ImageCaches(c.fledgedNameSpace).List(labels.Everything()) - if err != nil { - glog.Errorf("Error in listing image caches: %v", err) - return - } - for i := range imageCaches { - // Do not refresh if status is not yet updated - if reflect.DeepEqual(imageCaches[i].Status, fledgedv1alpha1.ImageCacheStatus{}) { - continue - } - // Do not refresh if image cache is already under processing - if imageCaches[i].Status.Status == fledgedv1alpha1.ImageCacheActionStatusProcessing { - continue - } - // Do not refresh image cache if cache spec validation failed - if imageCaches[i].Status.Status == fledgedv1alpha1.ImageCacheActionStatusFailed && - imageCaches[i].Status.Reason == fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed { - continue - } - // Do not refresh if image cache has been purged - if imageCaches[i].Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCachePurge { - continue - } - c.enqueueImageCache(images.ImageCacheRefresh, imageCaches[i], nil) - } -} - -// syncHandler compares the actual state with the desired, and attempts to -// converge the two. It then updates the Status block of the ImageCache resource -// with the current status of the resource. -func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error { - status := &fledgedv1alpha1.ImageCacheStatus{ - Failures: map[string]fledgedv1alpha1.NodeReasonMessageList{}, - } - - // Convert the namespace/name string into a distinct namespace and name - namespace, name, err := cache.SplitMetaNamespaceKey(wqKey.ObjKey) - if err != nil { - glog.Errorf("Error from cache.SplitMetaNamespaceKey(): %v", err) - return err - } - - glog.Infof("Starting to sync image cache %s(%s)", name, wqKey.WorkType) - - switch wqKey.WorkType { - case images.ImageCacheCreate, images.ImageCacheUpdate, images.ImageCacheRefresh, images.ImageCachePurge: - - startTime := metav1.Now() - status.StartTime = &startTime - // Get the ImageCache resource with this namespace/name - imageCache, err := c.imageCachesLister.ImageCaches(namespace).Get(name) - if err != nil { - // The ImageCache resource may no longer exist, in which case we stop - // processing. - glog.Errorf("Error getting imagecache(%s): %v", name, err) - return err - } - - err = validateCacheSpec(c, imageCache) - if err != nil { - status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed - status.Reason = fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed - status.Message = err.Error() - - if err := c.updateImageCacheStatus(imageCache, status); err != nil { - glog.Errorf("Error updating imagecache status to %s: %v", status.Status, err) - return err - } - - return err - } - - if wqKey.WorkType == images.ImageCacheUpdate && wqKey.OldImageCache == nil { - status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed - status.Reason = fledgedv1alpha1.ImageCacheReasonOldImageCacheNotFound - status.Message = fledgedv1alpha1.ImageCacheMessageOldImageCacheNotFound - - if err := c.updateImageCacheStatus(imageCache, status); err != nil { - glog.Errorf("Error updating imagecache status to %s: %v", status.Status, err) - return err - } - glog.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonOldImageCacheNotFound, fledgedv1alpha1.ImageCacheMessageOldImageCacheNotFound) - return fmt.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonOldImageCacheNotFound, fledgedv1alpha1.ImageCacheMessageOldImageCacheNotFound) - } - - if wqKey.WorkType == images.ImageCacheUpdate { - if len(wqKey.OldImageCache.Spec.CacheSpec) != len(imageCache.Spec.CacheSpec) { - status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed - status.Reason = fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed - status.Message = fledgedv1alpha1.ImageCacheMessageNotSupportedUpdates - - if err = c.updateImageCacheSpecAndStatus(imageCache, wqKey.OldImageCache.Spec, status); err != nil { - glog.Errorf("Error updating imagecache spec and status to %s: %v", status.Status, err) - return err - } - glog.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, "Mismatch in no. of image lists") - return fmt.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, "Mismatch in no. of image lists") - } - - for i := range wqKey.OldImageCache.Spec.CacheSpec { - if !reflect.DeepEqual(wqKey.OldImageCache.Spec.CacheSpec[i].NodeSelector, imageCache.Spec.CacheSpec[i].NodeSelector) { - status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed - status.Reason = fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed - status.Message = fledgedv1alpha1.ImageCacheMessageNotSupportedUpdates - - if err = c.updateImageCacheSpecAndStatus(imageCache, wqKey.OldImageCache.Spec, status); err != nil { - glog.Errorf("Error updating imagecache spec and status to %s: %v", status.Status, err) - return err - } - glog.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, "Mismatch in node selector") - return fmt.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, "Mismatch in node selector") - } - } - } - - cacheSpec := imageCache.Spec.CacheSpec - glog.V(4).Infof("cacheSpec: %+v", cacheSpec) - var nodes []*corev1.Node - - status.Status = fledgedv1alpha1.ImageCacheActionStatusProcessing - - if wqKey.WorkType == images.ImageCacheCreate { - status.Reason = fledgedv1alpha1.ImageCacheReasonImageCacheCreate - status.Message = fledgedv1alpha1.ImageCacheMessagePullingImages - } - - if wqKey.WorkType == images.ImageCacheUpdate { - status.Reason = fledgedv1alpha1.ImageCacheReasonImageCacheUpdate - status.Message = fledgedv1alpha1.ImageCacheMessageUpdatingCache - } - - if wqKey.WorkType == images.ImageCacheRefresh { - status.Reason = fledgedv1alpha1.ImageCacheReasonImageCacheRefresh - status.Message = fledgedv1alpha1.ImageCacheMessageRefreshingCache - } - - if wqKey.WorkType == images.ImageCachePurge { - status.Reason = fledgedv1alpha1.ImageCacheReasonImageCachePurge - status.Message = fledgedv1alpha1.ImageCacheMessagePurgeCache - } - - imageCache, err = c.fledgedclientset.FledgedV1alpha1().ImageCaches(namespace).Get(name, metav1.GetOptions{}) - if err != nil { - glog.Errorf("Error getting imagecache(%s) from api server: %v", name, err) - return err - } - - if err = c.updateImageCacheStatus(imageCache, status); err != nil { - glog.Errorf("Error updating imagecache status to %s: %v", status.Status, err) - return err - } - - for k, i := range cacheSpec { - if len(i.NodeSelector) > 0 { - if nodes, err = c.nodesLister.List(labels.Set(i.NodeSelector).AsSelector()); err != nil { - glog.Errorf("Error listing nodes using nodeselector %+v: %v", i.NodeSelector, err) - return err - } - } else { - if nodes, err = c.nodesLister.List(labels.Everything()); err != nil { - glog.Errorf("Error listing nodes using nodeselector labels.Everything(): %v", err) - return err - } - } - glog.V(4).Infof("No. of nodes in %+v is %d", i.NodeSelector, len(nodes)) - if len(nodes) == 0 { - glog.Errorf("NodeSelector %+v did not match any nodes.", i.NodeSelector) - return fmt.Errorf("NodeSelector %+v did not match any nodes", i.NodeSelector) - } - - for _, n := range nodes { - for m := range i.Images { - ipr := images.ImageWorkRequest{ - Image: i.Images[m], - Node: n, - ContainerRuntimeVersion: n.Status.NodeInfo.ContainerRuntimeVersion, - WorkType: wqKey.WorkType, - Imagecache: imageCache, - } - c.imageworkqueue.AddRateLimited(ipr) - } - if wqKey.WorkType == images.ImageCacheUpdate { - for _, oldimage := range wqKey.OldImageCache.Spec.CacheSpec[k].Images { - matched := false - for _, newimage := range i.Images { - if oldimage == newimage { - matched = true - break - } - } - if !matched { - ipr := images.ImageWorkRequest{ - Image: oldimage, - Node: n, - ContainerRuntimeVersion: n.Status.NodeInfo.ContainerRuntimeVersion, - WorkType: images.ImageCachePurge, - Imagecache: imageCache, - } - c.imageworkqueue.AddRateLimited(ipr) - } - } - } - } - } - - // We add an empty image pull request to signal the image manager that all - // requests for this sync action have been placed in the imageworkqueue - c.imageworkqueue.AddRateLimited(images.ImageWorkRequest{WorkType: wqKey.WorkType, Imagecache: imageCache}) - - case images.ImageCacheStatusUpdate: - glog.V(4).Infof("wqKey.Status = %+v", wqKey.Status) - // Finally, we update the status block of the ImageCache resource to reflect the - // current state of the world - // Get the ImageCache resource with this namespace/name - imageCache, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(namespace).Get(name, metav1.GetOptions{}) - if err != nil { - glog.Errorf("Error getting image cache %s: %v", name, err) - return err - } - - if imageCache.Status.StartTime != nil { - status.StartTime = imageCache.Status.StartTime - } - - status.Reason = imageCache.Status.Reason - - failures := false - for _, v := range *wqKey.Status { - if (v.Status == images.ImageWorkResultStatusSucceeded || v.Status == images.ImageWorkResultStatusAlreadyPulled) && !failures { - status.Status = fledgedv1alpha1.ImageCacheActionStatusSucceeded - if v.ImageWorkRequest.WorkType == images.ImageCachePurge { - status.Message = fledgedv1alpha1.ImageCacheMessageImagesDeletedSuccessfully - } else { - status.Message = fledgedv1alpha1.ImageCacheMessageImagesPulledSuccessfully - } - } - if v.Status == images.ImageWorkResultStatusFailed && !failures { - failures = true - status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed - if v.ImageWorkRequest.WorkType == images.ImageCachePurge { - status.Message = fledgedv1alpha1.ImageCacheMessageImageDeleteFailedForSomeImages - } else { - status.Message = fledgedv1alpha1.ImageCacheMessageImagePullFailedForSomeImages - } - } - if v.Status == images.ImageWorkResultStatusFailed { - status.Failures[v.ImageWorkRequest.Image] = append( - status.Failures[v.ImageWorkRequest.Image], fledgedv1alpha1.NodeReasonMessage{ - Node: v.ImageWorkRequest.Node.Labels["kubernetes.io/hostname"], - Reason: v.Reason, - Message: v.Message, - }) - } - } - - err = c.updateImageCacheStatus(imageCache, status) - if err != nil { - glog.Errorf("Error updating ImageCache status: %v", err) - return err - } - - if imageCache.Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCachePurge || imageCache.Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCacheRefresh { - imageCache, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(namespace).Get(name, metav1.GetOptions{}) - if err != nil { - glog.Errorf("Error getting image cache %s: %v", name, err) - return err - } - if imageCache.Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCachePurge { - if err := c.removeAnnotation(imageCache, imageCachePurgeAnnotationKey); err != nil { - glog.Errorf("Error removing Annotation %s from imagecache(%s): %v", imageCachePurgeAnnotationKey, imageCache.Name, err) - return err - } - } - if imageCache.Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCacheRefresh { - if err := c.removeAnnotation(imageCache, imageCacheRefreshAnnotationKey); err != nil { - glog.Errorf("Error removing Annotation %s from imagecache(%s): %v", imageCacheRefreshAnnotationKey, imageCache.Name, err) - return err - } - } - } - - if status.Status == fledgedv1alpha1.ImageCacheActionStatusSucceeded { - c.recorder.Event(imageCache, corev1.EventTypeNormal, status.Reason, status.Message) - } - - if status.Status == fledgedv1alpha1.ImageCacheActionStatusFailed { - c.recorder.Event(imageCache, corev1.EventTypeWarning, status.Reason, status.Message) - } - } - glog.Infof("Completed sync actions for image cache %s(%s)", name, wqKey.WorkType) - return nil - -} - -func (c *Controller) updateImageCacheStatus(imageCache *fledgedv1alpha1.ImageCache, status *fledgedv1alpha1.ImageCacheStatus) error { - // NEVER modify objects from the store. It's a read-only, local cache. - // You can use DeepCopy() to make a deep copy of original object and modify this copy - // Or create a copy manually for better performance - imageCacheCopy := imageCache.DeepCopy() - imageCacheCopy.Status = *status - if imageCacheCopy.Status.Status != fledgedv1alpha1.ImageCacheActionStatusProcessing { - completionTime := metav1.Now() - imageCacheCopy.Status.CompletionTime = &completionTime - } - // If the CustomResourceSubresources feature gate is not enabled, - // we must use Update instead of UpdateStatus to update the Status block of the ImageCache resource. - // UpdateStatus will not allow changes to the Spec of the resource, - // which is ideal for ensuring nothing other than resource status has been updated. - _, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(imageCache.Namespace).Update(imageCacheCopy) - return err -} - -func (c *Controller) updateImageCacheSpecAndStatus(imageCache *fledgedv1alpha1.ImageCache, spec fledgedv1alpha1.ImageCacheSpec, status *fledgedv1alpha1.ImageCacheStatus) error { - // NEVER modify objects from the store. It's a read-only, local cache. - // You can use DeepCopy() to make a deep copy of original object and modify this copy - // Or create a copy manually for better performance - imageCacheCopy := imageCache.DeepCopy() - imageCacheCopy.Spec = spec - imageCacheCopy.Status = *status - - if status.Status == fledgedv1alpha1.ImageCacheActionStatusFailed && - status.Reason == fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed { - imageCacheCopy.Annotations = make(map[string]string) - imageCacheCopy.Annotations[fledgedCacheSpecValidationKey] = "failed" - } - - if imageCacheCopy.Status.Status != fledgedv1alpha1.ImageCacheActionStatusProcessing { - completionTime := metav1.Now() - imageCacheCopy.Status.CompletionTime = &completionTime - } - // If the CustomResourceSubresources feature gate is not enabled, - // we must use Update instead of UpdateStatus to update the Status block of the ImageCache resource. - // UpdateStatus will not allow changes to the Spec of the resource, - // which is ideal for ensuring nothing other than resource status has been updated. - _, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(imageCache.Namespace).Update(imageCacheCopy) - return err -} - -func (c *Controller) removeAnnotation(imageCache *fledgedv1alpha1.ImageCache, annotationKey string) error { - imageCacheCopy := imageCache.DeepCopy() - delete(imageCacheCopy.Annotations, annotationKey) - _, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(imageCache.Namespace).Update(imageCacheCopy) - if err == nil { - glog.Infof("Annotation %s removed from imagecache(%s)", fledgedCacheSpecValidationKey, imageCache.Name) - } - return err -} +/* +Copyright 2018 The kube-fledged authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + "reflect" + "time" + + "github.com/golang/glog" + fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + clientset "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned" + fledgedscheme "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/scheme" + informers "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions/fledged/v1alpha1" + listers "github.com/senthilrch/kube-fledged/pkg/client/listers/fledged/v1alpha1" + "github.com/senthilrch/kube-fledged/pkg/images" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + coreinformers "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" +) + +const controllerAgentName = "fledged" +const fledgedCacheSpecValidationKey = "fledged.k8s.io/cachespecvalidation" +const imageCachePurgeAnnotationKey = "fledged.k8s.io/purge-imagecache" +const imageCacheRefreshAnnotationKey = "fledged.k8s.io/refresh-imagecache" + +const ( + // SuccessSynced is used as part of the Event 'reason' when a ImageCache is synced + SuccessSynced = "Synced" + // MessageResourceSynced is the message used for an Event fired when a ImageCache + // is synced successfully + MessageResourceSynced = "ImageCache synced successfully" +) + +// Controller is the controller for ImageCache resources +type Controller struct { + // kubeclientset is a standard kubernetes clientset + kubeclientset kubernetes.Interface + // fledgedclientset is a clientset for fledged.k8s.io API group + fledgedclientset clientset.Interface + + fledgedNameSpace string + nodesLister corelisters.NodeLister + nodesSynced cache.InformerSynced + imageCachesLister listers.ImageCacheLister + imageCachesSynced cache.InformerSynced + + // workqueue is a rate limited work queue. This is used to queue work to be + // processed instead of performing it as soon as a change happens. This + // means we can ensure we only process a fixed amount of resources at a + // time, and makes it easy to ensure we are never processing the same item + // simultaneously in two different workers. + workqueue workqueue.RateLimitingInterface + imageworkqueue workqueue.RateLimitingInterface + imageManager *images.ImageManager + // recorder is an event recorder for recording Event resources to the + // Kubernetes API. + recorder record.EventRecorder + imageCacheRefreshFrequency time.Duration +} + +// NewController returns a new fledged controller +func NewController( + kubeclientset kubernetes.Interface, + fledgedclientset clientset.Interface, + namespace string, + nodeInformer coreinformers.NodeInformer, + imageCacheInformer informers.ImageCacheInformer, + imageCacheRefreshFrequency time.Duration, + imagePullDeadlineDuration time.Duration, + dockerClientImage string, + imagePullPolicy string) *Controller { + + utilruntime.Must(fledgedscheme.AddToScheme(scheme.Scheme)) + glog.V(4).Info("Creating event broadcaster") + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: kubeclientset.CoreV1().Events("")}) + recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerAgentName}) + + controller := &Controller{ + kubeclientset: kubeclientset, + fledgedclientset: fledgedclientset, + fledgedNameSpace: namespace, + nodesLister: nodeInformer.Lister(), + nodesSynced: nodeInformer.Informer().HasSynced, + imageCachesLister: imageCacheInformer.Lister(), + imageCachesSynced: imageCacheInformer.Informer().HasSynced, + workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ImageCaches"), + imageworkqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ImagePullerStatus"), + recorder: recorder, + imageCacheRefreshFrequency: imageCacheRefreshFrequency, + } + + imageManager, _ := images.NewImageManager(controller.workqueue, controller.imageworkqueue, controller.kubeclientset, controller.fledgedNameSpace, imagePullDeadlineDuration, dockerClientImage, imagePullPolicy) + controller.imageManager = imageManager + + glog.Info("Setting up event handlers") + // Set up an event handler for when ImageCache resources change + imageCacheInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + controller.enqueueImageCache(images.ImageCacheCreate, nil, obj) + }, + UpdateFunc: func(old, new interface{}) { + controller.enqueueImageCache(images.ImageCacheUpdate, old, new) + }, + DeleteFunc: func(obj interface{}) { + controller.enqueueImageCache(images.ImageCacheDelete, obj, nil) + }, + }) + return controller +} + +// PreFlightChecks performs pre-flight checks and actions before the controller is started +func (c *Controller) PreFlightChecks() error { + if err := c.danglingJobs(); err != nil { + return err + } + if err := c.danglingImageCaches(); err != nil { + return err + } + return nil +} + +// danglingJobs finds and removes dangling or stuck jobs +func (c *Controller) danglingJobs() error { + joblist, err := c.kubeclientset.BatchV1().Jobs(c.fledgedNameSpace).List(metav1.ListOptions{}) + if err != nil { + glog.Errorf("Error listing jobs: %v", err) + return err + } + + if joblist == nil || len(joblist.Items) == 0 { + glog.Info("No dangling or stuck jobs found...") + return nil + } + deletePropagation := metav1.DeletePropagationBackground + for _, job := range joblist.Items { + err := c.kubeclientset.BatchV1().Jobs(c.fledgedNameSpace). + Delete(job.Name, &metav1.DeleteOptions{PropagationPolicy: &deletePropagation}) + if err != nil { + glog.Errorf("Error deleting job(%s): %v", job.Name, err) + return err + } + glog.Infof("Dangling Job(%s) deleted", job.Name) + } + return nil +} + +// danglingImageCaches finds dangling or stuck image cache and marks them as abhorted. Such +// image caches will get refreshed in the next cycle +func (c *Controller) danglingImageCaches() error { + dangling := false + imagecachelist, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(c.fledgedNameSpace).List(metav1.ListOptions{}) + if err != nil { + glog.Errorf("Error listing imagecaches: %v", err) + return err + } + + if imagecachelist == nil || len(imagecachelist.Items) == 0 { + glog.Info("No dangling or stuck imagecaches found...") + return nil + } + status := &fledgedv1alpha1.ImageCacheStatus{ + Failures: map[string]fledgedv1alpha1.NodeReasonMessageList{}, + Status: fledgedv1alpha1.ImageCacheActionStatusAborted, + Reason: fledgedv1alpha1.ImageCacheReasonImagePullAborted, + Message: fledgedv1alpha1.ImageCacheMessageImagePullAborted, + } + for _, imagecache := range imagecachelist.Items { + if imagecache.Status.Status == fledgedv1alpha1.ImageCacheActionStatusProcessing { + status.StartTime = imagecache.Status.StartTime + err := c.updateImageCacheStatus(&imagecache, status) + if err != nil { + glog.Errorf("Error updating ImageCache(%s) status to '%s': %v", imagecache.Name, fledgedv1alpha1.ImageCacheActionStatusAborted, err) + return err + } + dangling = true + glog.Infof("Dangling Image cache(%s) status changed to '%s'", imagecache.Name, fledgedv1alpha1.ImageCacheActionStatusAborted) + } + } + + if !dangling { + glog.Info("No dangling or stuck imagecaches found...") + } + return nil +} + +// Run will set up the event handlers for types we are interested in, as well +// as syncing informer caches and starting workers. It will block until stopCh +// is closed, at which point it will shutdown the workqueue and wait for +// workers to finish processing their current work items. +func (c *Controller) Run(threadiness int, stopCh <-chan struct{}) error { + defer runtime.HandleCrash() + defer c.workqueue.ShutDown() + defer c.imageworkqueue.ShutDown() + + // Start the informer factories to begin populating the informer caches + glog.Info("Starting fledged controller") + + // Wait for the caches to be synced before starting workers + glog.Info("Waiting for informer caches to sync") + if ok := cache.WaitForCacheSync(stopCh, c.nodesSynced, c.imageCachesSynced); !ok { + return fmt.Errorf("failed to wait for caches to sync") + } + + glog.Info("Starting image cache worker") + // Launch workers to process ImageCache resources + for i := 0; i < threadiness; i++ { + go wait.Until(c.runWorker, time.Second, stopCh) + } + + if c.imageCacheRefreshFrequency.Nanoseconds() != int64(0) { + glog.Info("Starting cache refresh worker") + go wait.Until(c.runRefreshWorker, c.imageCacheRefreshFrequency, stopCh) + } + + glog.Info("Started workers") + c.imageManager.Run(stopCh) + if err := c.imageManager.Run(stopCh); err != nil { + glog.Fatalf("Error running image manager: %s", err.Error()) + } + + <-stopCh + glog.Info("Shutting down workers") + + return nil +} + +// enqueueImageCache takes a ImageCache resource and converts it into a namespace/name +// string which is then put onto the work queue. This method should *not* be +// passed resources of any type other than ImageCache. +func (c *Controller) enqueueImageCache(workType images.WorkType, old, new interface{}) bool { + var key string + var err error + var obj interface{} + wqKey := images.WorkQueueKey{} + + switch workType { + case images.ImageCacheCreate: + obj = new + newImageCache := new.(*fledgedv1alpha1.ImageCache) + // If the ImageCache resource already has a status field, it means it's already + // synced, so do not queue it for processing + if !reflect.DeepEqual(newImageCache.Status, fledgedv1alpha1.ImageCacheStatus{}) { + return false + } + case images.ImageCacheUpdate: + obj = new + oldImageCache := old.(*fledgedv1alpha1.ImageCache) + newImageCache := new.(*fledgedv1alpha1.ImageCache) + + if oldImageCache.Status.Status == fledgedv1alpha1.ImageCacheActionStatusProcessing { + if !reflect.DeepEqual(newImageCache.Spec, oldImageCache.Spec) { + glog.Warningf("Received image cache update/purge/delete for '%s' while it is under processing, so ignoring.", oldImageCache.Name) + return false + } + } + if _, exists := newImageCache.Annotations[imageCachePurgeAnnotationKey]; exists { + if _, exists := oldImageCache.Annotations[imageCachePurgeAnnotationKey]; !exists { + workType = images.ImageCachePurge + break + } + } + if _, exists := newImageCache.Annotations[imageCacheRefreshAnnotationKey]; exists { + if _, exists := oldImageCache.Annotations[imageCacheRefreshAnnotationKey]; !exists { + workType = images.ImageCacheRefresh + break + } + } + if !reflect.DeepEqual(newImageCache.Spec, oldImageCache.Spec) { + if validation, ok := newImageCache.Annotations[fledgedCacheSpecValidationKey]; ok { + if validation == "failed" { + if err := c.removeAnnotation(newImageCache, fledgedCacheSpecValidationKey); err != nil { + glog.Errorf("Error removing Annotation %s from imagecache(%s): %v", fledgedCacheSpecValidationKey, newImageCache.Name, err) + } + return false + } + } + } else { + return false + } + case images.ImageCacheDelete: + return false + + case images.ImageCacheRefresh: + obj = old + } + + if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { + runtime.HandleError(err) + return false + } + wqKey.WorkType = workType + wqKey.ObjKey = key + if workType == images.ImageCacheUpdate { + oldImageCache := old.(*fledgedv1alpha1.ImageCache) + wqKey.OldImageCache = oldImageCache + } + + c.workqueue.AddRateLimited(wqKey) + glog.V(4).Infof("enqueueImageCache::ImageCache resource queued for work type %s", workType) + return true +} + +// runWorker is a long-running function that will continually call the +// processNextWorkItem function in order to read and process a message on the +// workqueue. +func (c *Controller) runWorker() { + for c.processNextWorkItem() { + } +} + +// processNextWorkItem will read a single work item off the workqueue and +// attempt to process it, by calling the syncHandler. +func (c *Controller) processNextWorkItem() bool { + //glog.Info("processNextWorkItem::Beginning...") + obj, shutdown := c.workqueue.Get() + + if shutdown { + return false + } + + // We wrap this block in a func so we can defer c.workqueue.Done. + err := func(obj interface{}) error { + // We call Done here so the workqueue knows we have finished + // processing this item. We also must remember to call Forget if we + // do not want this work item being re-queued. For example, we do + // not call Forget if a transient error occurs, instead the item is + // put back on the workqueue and attempted again after a back-off + // period. + defer c.workqueue.Done(obj) + var key images.WorkQueueKey + var ok bool + // We expect strings to come off the workqueue. These are of the + // form namespace/name. We do this as the delayed nature of the + // workqueue means the items in the informer cache may actually be + // more up to date that when the item was initially put onto the + // workqueue. + if key, ok = obj.(images.WorkQueueKey); !ok { + // As the item in the workqueue is actually invalid, we call + // Forget here else we'd go into a loop of attempting to + // process a work item that is invalid. + c.workqueue.Forget(obj) + runtime.HandleError(fmt.Errorf("Unexpected type in workqueue: %#v", obj)) + return nil + } + // Run the syncHandler, passing it the namespace/name string of the + // ImageCache resource to be synced. + if err := c.syncHandler(key); err != nil { + glog.Errorf("error syncing imagecache: %v", err.Error()) + return fmt.Errorf("error syncing imagecache: %v", err.Error()) + } + // Finally, if no error occurs we Forget this item so it does not + // get queued again until another change happens. + c.workqueue.Forget(obj) + //glog.Infof("Successfully synced '%s' for event '%s'", key.ObjKey, key.WorkType) + return nil + }(obj) + + if err != nil { + runtime.HandleError(err) + return true + } + + return true +} + +// runRefreshWorker is resposible of refreshing the image cache +func (c *Controller) runRefreshWorker() { + // List the ImageCache resources + imageCaches, err := c.imageCachesLister.ImageCaches(c.fledgedNameSpace).List(labels.Everything()) + if err != nil { + glog.Errorf("Error in listing image caches: %v", err) + return + } + for i := range imageCaches { + // Do not refresh if status is not yet updated + if reflect.DeepEqual(imageCaches[i].Status, fledgedv1alpha1.ImageCacheStatus{}) { + continue + } + // Do not refresh if image cache is already under processing + if imageCaches[i].Status.Status == fledgedv1alpha1.ImageCacheActionStatusProcessing { + continue + } + // Do not refresh image cache if cache spec validation failed + if imageCaches[i].Status.Status == fledgedv1alpha1.ImageCacheActionStatusFailed && + imageCaches[i].Status.Reason == fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed { + continue + } + // Do not refresh if image cache has been purged + if imageCaches[i].Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCachePurge { + continue + } + c.enqueueImageCache(images.ImageCacheRefresh, imageCaches[i], nil) + } +} + +// syncHandler compares the actual state with the desired, and attempts to +// converge the two. It then updates the Status block of the ImageCache resource +// with the current status of the resource. +func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error { + status := &fledgedv1alpha1.ImageCacheStatus{ + Failures: map[string]fledgedv1alpha1.NodeReasonMessageList{}, + } + + // Convert the namespace/name string into a distinct namespace and name + namespace, name, err := cache.SplitMetaNamespaceKey(wqKey.ObjKey) + if err != nil { + glog.Errorf("Error from cache.SplitMetaNamespaceKey(): %v", err) + return err + } + + glog.Infof("Starting to sync image cache %s(%s)", name, wqKey.WorkType) + + switch wqKey.WorkType { + case images.ImageCacheCreate, images.ImageCacheUpdate, images.ImageCacheRefresh, images.ImageCachePurge: + + startTime := metav1.Now() + status.StartTime = &startTime + // Get the ImageCache resource with this namespace/name + imageCache, err := c.imageCachesLister.ImageCaches(namespace).Get(name) + if err != nil { + // The ImageCache resource may no longer exist, in which case we stop + // processing. + glog.Errorf("Error getting imagecache(%s): %v", name, err) + return err + } + + err = validateCacheSpec(c, imageCache) + if err != nil { + status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed + status.Reason = fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed + status.Message = err.Error() + + if err := c.updateImageCacheStatus(imageCache, status); err != nil { + glog.Errorf("Error updating imagecache status to %s: %v", status.Status, err) + return err + } + + return err + } + + if wqKey.WorkType == images.ImageCacheUpdate && wqKey.OldImageCache == nil { + status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed + status.Reason = fledgedv1alpha1.ImageCacheReasonOldImageCacheNotFound + status.Message = fledgedv1alpha1.ImageCacheMessageOldImageCacheNotFound + + if err := c.updateImageCacheStatus(imageCache, status); err != nil { + glog.Errorf("Error updating imagecache status to %s: %v", status.Status, err) + return err + } + glog.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonOldImageCacheNotFound, fledgedv1alpha1.ImageCacheMessageOldImageCacheNotFound) + return fmt.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonOldImageCacheNotFound, fledgedv1alpha1.ImageCacheMessageOldImageCacheNotFound) + } + + if wqKey.WorkType == images.ImageCacheUpdate { + if len(wqKey.OldImageCache.Spec.CacheSpec) != len(imageCache.Spec.CacheSpec) { + status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed + status.Reason = fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed + status.Message = fledgedv1alpha1.ImageCacheMessageNotSupportedUpdates + + if err = c.updateImageCacheSpecAndStatus(imageCache, wqKey.OldImageCache.Spec, status); err != nil { + glog.Errorf("Error updating imagecache spec and status to %s: %v", status.Status, err) + return err + } + glog.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, "Mismatch in no. of image lists") + return fmt.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, "Mismatch in no. of image lists") + } + + for i := range wqKey.OldImageCache.Spec.CacheSpec { + if !reflect.DeepEqual(wqKey.OldImageCache.Spec.CacheSpec[i].NodeSelector, imageCache.Spec.CacheSpec[i].NodeSelector) { + status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed + status.Reason = fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed + status.Message = fledgedv1alpha1.ImageCacheMessageNotSupportedUpdates + + if err = c.updateImageCacheSpecAndStatus(imageCache, wqKey.OldImageCache.Spec, status); err != nil { + glog.Errorf("Error updating imagecache spec and status to %s: %v", status.Status, err) + return err + } + glog.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, "Mismatch in node selector") + return fmt.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, "Mismatch in node selector") + } + } + } + + cacheSpec := imageCache.Spec.CacheSpec + glog.V(4).Infof("cacheSpec: %+v", cacheSpec) + var nodes []*corev1.Node + + status.Status = fledgedv1alpha1.ImageCacheActionStatusProcessing + + if wqKey.WorkType == images.ImageCacheCreate { + status.Reason = fledgedv1alpha1.ImageCacheReasonImageCacheCreate + status.Message = fledgedv1alpha1.ImageCacheMessagePullingImages + } + + if wqKey.WorkType == images.ImageCacheUpdate { + status.Reason = fledgedv1alpha1.ImageCacheReasonImageCacheUpdate + status.Message = fledgedv1alpha1.ImageCacheMessageUpdatingCache + } + + if wqKey.WorkType == images.ImageCacheRefresh { + status.Reason = fledgedv1alpha1.ImageCacheReasonImageCacheRefresh + status.Message = fledgedv1alpha1.ImageCacheMessageRefreshingCache + } + + if wqKey.WorkType == images.ImageCachePurge { + status.Reason = fledgedv1alpha1.ImageCacheReasonImageCachePurge + status.Message = fledgedv1alpha1.ImageCacheMessagePurgeCache + } + + imageCache, err = c.fledgedclientset.FledgedV1alpha1().ImageCaches(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + glog.Errorf("Error getting imagecache(%s) from api server: %v", name, err) + return err + } + + if err = c.updateImageCacheStatus(imageCache, status); err != nil { + glog.Errorf("Error updating imagecache status to %s: %v", status.Status, err) + return err + } + + for k, i := range cacheSpec { + if len(i.NodeSelector) > 0 { + if nodes, err = c.nodesLister.List(labels.Set(i.NodeSelector).AsSelector()); err != nil { + glog.Errorf("Error listing nodes using nodeselector %+v: %v", i.NodeSelector, err) + return err + } + } else { + if nodes, err = c.nodesLister.List(labels.Everything()); err != nil { + glog.Errorf("Error listing nodes using nodeselector labels.Everything(): %v", err) + return err + } + } + glog.V(4).Infof("No. of nodes in %+v is %d", i.NodeSelector, len(nodes)) + if len(nodes) == 0 { + glog.Errorf("NodeSelector %+v did not match any nodes.", i.NodeSelector) + return fmt.Errorf("NodeSelector %+v did not match any nodes", i.NodeSelector) + } + + for _, n := range nodes { + for m := range i.Images { + ipr := images.ImageWorkRequest{ + Image: i.Images[m], + Node: n, + ContainerRuntimeVersion: n.Status.NodeInfo.ContainerRuntimeVersion, + WorkType: wqKey.WorkType, + Imagecache: imageCache, + } + c.imageworkqueue.AddRateLimited(ipr) + } + if wqKey.WorkType == images.ImageCacheUpdate { + for _, oldimage := range wqKey.OldImageCache.Spec.CacheSpec[k].Images { + matched := false + for _, newimage := range i.Images { + if oldimage == newimage { + matched = true + break + } + } + if !matched { + ipr := images.ImageWorkRequest{ + Image: oldimage, + Node: n, + ContainerRuntimeVersion: n.Status.NodeInfo.ContainerRuntimeVersion, + WorkType: images.ImageCachePurge, + Imagecache: imageCache, + } + c.imageworkqueue.AddRateLimited(ipr) + } + } + } + } + } + + // We add an empty image pull request to signal the image manager that all + // requests for this sync action have been placed in the imageworkqueue + c.imageworkqueue.AddRateLimited(images.ImageWorkRequest{WorkType: wqKey.WorkType, Imagecache: imageCache}) + + case images.ImageCacheStatusUpdate: + glog.V(4).Infof("wqKey.Status = %+v", wqKey.Status) + // Finally, we update the status block of the ImageCache resource to reflect the + // current state of the world + // Get the ImageCache resource with this namespace/name + imageCache, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + glog.Errorf("Error getting image cache %s: %v", name, err) + return err + } + + if imageCache.Status.StartTime != nil { + status.StartTime = imageCache.Status.StartTime + } + + status.Reason = imageCache.Status.Reason + + failures := false + for _, v := range *wqKey.Status { + if (v.Status == images.ImageWorkResultStatusSucceeded || v.Status == images.ImageWorkResultStatusAlreadyPulled) && !failures { + status.Status = fledgedv1alpha1.ImageCacheActionStatusSucceeded + if v.ImageWorkRequest.WorkType == images.ImageCachePurge { + status.Message = fledgedv1alpha1.ImageCacheMessageImagesDeletedSuccessfully + } else { + status.Message = fledgedv1alpha1.ImageCacheMessageImagesPulledSuccessfully + } + } + if v.Status == images.ImageWorkResultStatusFailed && !failures { + failures = true + status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed + if v.ImageWorkRequest.WorkType == images.ImageCachePurge { + status.Message = fledgedv1alpha1.ImageCacheMessageImageDeleteFailedForSomeImages + } else { + status.Message = fledgedv1alpha1.ImageCacheMessageImagePullFailedForSomeImages + } + } + if v.Status == images.ImageWorkResultStatusFailed { + status.Failures[v.ImageWorkRequest.Image] = append( + status.Failures[v.ImageWorkRequest.Image], fledgedv1alpha1.NodeReasonMessage{ + Node: v.ImageWorkRequest.Node.Labels["kubernetes.io/hostname"], + Reason: v.Reason, + Message: v.Message, + }) + } + } + + err = c.updateImageCacheStatus(imageCache, status) + if err != nil { + glog.Errorf("Error updating ImageCache status: %v", err) + return err + } + + if imageCache.Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCachePurge || imageCache.Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCacheRefresh { + imageCache, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + glog.Errorf("Error getting image cache %s: %v", name, err) + return err + } + if imageCache.Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCachePurge { + if err := c.removeAnnotation(imageCache, imageCachePurgeAnnotationKey); err != nil { + glog.Errorf("Error removing Annotation %s from imagecache(%s): %v", imageCachePurgeAnnotationKey, imageCache.Name, err) + return err + } + } + if imageCache.Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCacheRefresh { + if err := c.removeAnnotation(imageCache, imageCacheRefreshAnnotationKey); err != nil { + glog.Errorf("Error removing Annotation %s from imagecache(%s): %v", imageCacheRefreshAnnotationKey, imageCache.Name, err) + return err + } + } + } + + if status.Status == fledgedv1alpha1.ImageCacheActionStatusSucceeded { + c.recorder.Event(imageCache, corev1.EventTypeNormal, status.Reason, status.Message) + } + + if status.Status == fledgedv1alpha1.ImageCacheActionStatusFailed { + c.recorder.Event(imageCache, corev1.EventTypeWarning, status.Reason, status.Message) + } + } + glog.Infof("Completed sync actions for image cache %s(%s)", name, wqKey.WorkType) + return nil + +} + +func (c *Controller) updateImageCacheStatus(imageCache *fledgedv1alpha1.ImageCache, status *fledgedv1alpha1.ImageCacheStatus) error { + // NEVER modify objects from the store. It's a read-only, local cache. + // You can use DeepCopy() to make a deep copy of original object and modify this copy + // Or create a copy manually for better performance + imageCacheCopy := imageCache.DeepCopy() + imageCacheCopy.Status = *status + if imageCacheCopy.Status.Status != fledgedv1alpha1.ImageCacheActionStatusProcessing { + completionTime := metav1.Now() + imageCacheCopy.Status.CompletionTime = &completionTime + } + // If the CustomResourceSubresources feature gate is not enabled, + // we must use Update instead of UpdateStatus to update the Status block of the ImageCache resource. + // UpdateStatus will not allow changes to the Spec of the resource, + // which is ideal for ensuring nothing other than resource status has been updated. + _, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(imageCache.Namespace).Update(imageCacheCopy) + return err +} + +func (c *Controller) updateImageCacheSpecAndStatus(imageCache *fledgedv1alpha1.ImageCache, spec fledgedv1alpha1.ImageCacheSpec, status *fledgedv1alpha1.ImageCacheStatus) error { + // NEVER modify objects from the store. It's a read-only, local cache. + // You can use DeepCopy() to make a deep copy of original object and modify this copy + // Or create a copy manually for better performance + imageCacheCopy := imageCache.DeepCopy() + imageCacheCopy.Spec = spec + imageCacheCopy.Status = *status + + if status.Status == fledgedv1alpha1.ImageCacheActionStatusFailed && + status.Reason == fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed { + imageCacheCopy.Annotations = make(map[string]string) + imageCacheCopy.Annotations[fledgedCacheSpecValidationKey] = "failed" + } + + if imageCacheCopy.Status.Status != fledgedv1alpha1.ImageCacheActionStatusProcessing { + completionTime := metav1.Now() + imageCacheCopy.Status.CompletionTime = &completionTime + } + // If the CustomResourceSubresources feature gate is not enabled, + // we must use Update instead of UpdateStatus to update the Status block of the ImageCache resource. + // UpdateStatus will not allow changes to the Spec of the resource, + // which is ideal for ensuring nothing other than resource status has been updated. + _, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(imageCache.Namespace).Update(imageCacheCopy) + return err +} + +func (c *Controller) removeAnnotation(imageCache *fledgedv1alpha1.ImageCache, annotationKey string) error { + imageCacheCopy := imageCache.DeepCopy() + delete(imageCacheCopy.Annotations, annotationKey) + _, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(imageCache.Namespace).Update(imageCacheCopy) + if err == nil { + glog.Infof("Annotation %s removed from imagecache(%s)", fledgedCacheSpecValidationKey, imageCache.Name) + } + return err +} diff --git a/cmd/fledged/app/controller_helpers.go b/cmd/fledged/app/controller_helpers.go index c8fa5ea0..dbe08fbf 100644 --- a/cmd/fledged/app/controller_helpers.go +++ b/cmd/fledged/app/controller_helpers.go @@ -1,73 +1,73 @@ -/* -Copyright 2018 The kube-fledged authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "fmt" - - "github.com/golang/glog" - fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" -) - -func validateCacheSpec(c *Controller, imageCache *fledgedv1alpha1.ImageCache) error { - if imageCache == nil { - glog.Errorf("Unable to obtain reference to image cache") - return fmt.Errorf("Unable to obtain reference to image cache") - } - - cacheSpec := imageCache.Spec.CacheSpec - glog.V(4).Infof("cacheSpec: %+v", cacheSpec) - var nodes []*corev1.Node - var err error - - for _, i := range cacheSpec { - if len(i.Images) == 0 { - glog.Error("No images specified within image list") - return fmt.Errorf("No images specified within image list") - - } - - for m := range i.Images { - for p := 0; p < m; p++ { - if i.Images[p] == i.Images[m] { - glog.Errorf("Duplicate image names within image list: %s", i.Images[m]) - return fmt.Errorf("Duplicate image names within image list: %s", i.Images[m]) - } - } - } - - if len(i.NodeSelector) > 0 { - if nodes, err = c.nodesLister.List(labels.Set(i.NodeSelector).AsSelector()); err != nil { - glog.Errorf("Error listing nodes using nodeselector %+v: %v", i.NodeSelector, err) - return err - } - } else { - if nodes, err = c.nodesLister.List(labels.Everything()); err != nil { - glog.Errorf("Error listing nodes using nodeselector labels.Everything(): %v", err) - return err - } - } - glog.V(4).Infof("No. of nodes in %+v is %d", i.NodeSelector, len(nodes)) - if len(nodes) == 0 { - glog.Errorf("NodeSelector %s did not match any nodes.", labels.Set(i.NodeSelector).String()) - return fmt.Errorf("NodeSelector %s did not match any nodes", labels.Set(i.NodeSelector).String()) - } - } - return nil -} +/* +Copyright 2018 The kube-fledged authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + + "github.com/golang/glog" + fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" +) + +func validateCacheSpec(c *Controller, imageCache *fledgedv1alpha1.ImageCache) error { + if imageCache == nil { + glog.Errorf("Unable to obtain reference to image cache") + return fmt.Errorf("Unable to obtain reference to image cache") + } + + cacheSpec := imageCache.Spec.CacheSpec + glog.V(4).Infof("cacheSpec: %+v", cacheSpec) + var nodes []*corev1.Node + var err error + + for _, i := range cacheSpec { + if len(i.Images) == 0 { + glog.Error("No images specified within image list") + return fmt.Errorf("No images specified within image list") + + } + + for m := range i.Images { + for p := 0; p < m; p++ { + if i.Images[p] == i.Images[m] { + glog.Errorf("Duplicate image names within image list: %s", i.Images[m]) + return fmt.Errorf("Duplicate image names within image list: %s", i.Images[m]) + } + } + } + + if len(i.NodeSelector) > 0 { + if nodes, err = c.nodesLister.List(labels.Set(i.NodeSelector).AsSelector()); err != nil { + glog.Errorf("Error listing nodes using nodeselector %+v: %v", i.NodeSelector, err) + return err + } + } else { + if nodes, err = c.nodesLister.List(labels.Everything()); err != nil { + glog.Errorf("Error listing nodes using nodeselector labels.Everything(): %v", err) + return err + } + } + glog.V(4).Infof("No. of nodes in %+v is %d", i.NodeSelector, len(nodes)) + if len(nodes) == 0 { + glog.Errorf("NodeSelector %s did not match any nodes.", labels.Set(i.NodeSelector).String()) + return fmt.Errorf("NodeSelector %s did not match any nodes", labels.Set(i.NodeSelector).String()) + } + } + return nil +} diff --git a/cmd/fledged/app/controller_helpers_test.go b/cmd/fledged/app/controller_helpers_test.go index 6c4d01f5..1c42c603 100644 --- a/cmd/fledged/app/controller_helpers_test.go +++ b/cmd/fledged/app/controller_helpers_test.go @@ -1,192 +1,192 @@ -/* -Copyright 2018 The kube-fledged authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "fmt" - "strings" - "testing" - - fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" - fledgedclientsetfake "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/fake" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - fakeclientset "k8s.io/client-go/kubernetes/fake" -) - -var ( - alwaysReady = func() bool { return true } -) - -func TestValidateCacheSpec(t *testing.T) { - //var fakekubeclientset *fakeclientset.Clientset - //var fakefledgedclientset *fledgedclientsetfake.Clientset - - tests := []struct { - name string - imageCache *fledgedv1alpha1.ImageCache - nodeList *corev1.NodeList - nodeListError error - expectErr bool - errorString string - }{ - { - name: "Unable to obtain reference to image cache", - imageCache: nil, - nodeList: nil, - nodeListError: nil, - expectErr: true, - errorString: "Unable to obtain reference to image cache", - }, - { - name: "No images specified within image list", - imageCache: &fledgedv1alpha1.ImageCache{ - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{}, - }, - }, - }, - }, - nodeList: nil, - nodeListError: nil, - expectErr: true, - errorString: "No images specified within image list", - }, - { - name: "Duplicate image names within image list", - imageCache: &fledgedv1alpha1.ImageCache{ - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo", "foo"}, - }, - }, - }, - }, - nodeList: nil, - nodeListError: nil, - expectErr: true, - errorString: "Duplicate image names within image list", - }, - { - name: "Error listing nodes using nodeselector", - imageCache: &fledgedv1alpha1.ImageCache{ - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - NodeSelector: map[string]string{"foo": "bar"}, - }, - }, - }, - }, - nodeList: nil, - nodeListError: fmt.Errorf("fake error"), - expectErr: true, - errorString: "Error listing nodes using nodeselector", - }, - { - name: "Error listing nodes using nodeselector labels.Everything()", - imageCache: &fledgedv1alpha1.ImageCache{ - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - }, - nodeList: nil, - nodeListError: fmt.Errorf("fake error"), - expectErr: true, - errorString: "Error listing nodes using nodeselector labels.Everything()", - }, - { - name: "NodeSelector did not match any nodes", - imageCache: &fledgedv1alpha1.ImageCache{ - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - NodeSelector: map[string]string{"foo": "bar"}, - }, - }, - }, - }, - nodeList: &corev1.NodeList{}, - nodeListError: nil, - expectErr: true, - errorString: "NodeSelector foo=bar did not match any nodes", - }, - { - name: "Successful validation", - imageCache: &fledgedv1alpha1.ImageCache{ - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - NodeSelector: map[string]string{"foo": "bar"}, - }, - }, - }, - }, - nodeList: &corev1.NodeList{ - Items: []corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "fakenode", - Labels: map[string]string{"foo": "bar"}, - }, - }, - }, - }, - nodeListError: nil, - expectErr: false, - errorString: "", - }, - } - - for _, test := range tests { - fakekubeclientset := &fakeclientset.Clientset{} - fakefledgedclientset := &fledgedclientsetfake.Clientset{} - - controller, nodeInformer, _ := newTestController(fakekubeclientset, fakefledgedclientset) - - if test.nodeListError != nil { - //TODO: How to return a fake error from node Lister? - continue - } - - if test.nodeList != nil && len(test.nodeList.Items) > 0 { - for _, node := range test.nodeList.Items { - nodeInformer.Informer().GetIndexer().Add(&node) - } - } - - err := validateCacheSpec(controller, test.imageCache) - if test.expectErr { - if err != nil && strings.HasPrefix(err.Error(), test.errorString) { - } else { - t.Errorf("Test: %s failed", test.name) - } - } else if err != nil { - t.Errorf("Test: %s failed. err received = %s", test.name, err.Error()) - } - } -} +/* +Copyright 2018 The kube-fledged authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + "strings" + "testing" + + fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + fledgedclientsetfake "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/fake" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + fakeclientset "k8s.io/client-go/kubernetes/fake" +) + +var ( + alwaysReady = func() bool { return true } +) + +func TestValidateCacheSpec(t *testing.T) { + //var fakekubeclientset *fakeclientset.Clientset + //var fakefledgedclientset *fledgedclientsetfake.Clientset + + tests := []struct { + name string + imageCache *fledgedv1alpha1.ImageCache + nodeList *corev1.NodeList + nodeListError error + expectErr bool + errorString string + }{ + { + name: "Unable to obtain reference to image cache", + imageCache: nil, + nodeList: nil, + nodeListError: nil, + expectErr: true, + errorString: "Unable to obtain reference to image cache", + }, + { + name: "No images specified within image list", + imageCache: &fledgedv1alpha1.ImageCache{ + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{}, + }, + }, + }, + }, + nodeList: nil, + nodeListError: nil, + expectErr: true, + errorString: "No images specified within image list", + }, + { + name: "Duplicate image names within image list", + imageCache: &fledgedv1alpha1.ImageCache{ + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo", "foo"}, + }, + }, + }, + }, + nodeList: nil, + nodeListError: nil, + expectErr: true, + errorString: "Duplicate image names within image list", + }, + { + name: "Error listing nodes using nodeselector", + imageCache: &fledgedv1alpha1.ImageCache{ + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + NodeSelector: map[string]string{"foo": "bar"}, + }, + }, + }, + }, + nodeList: nil, + nodeListError: fmt.Errorf("fake error"), + expectErr: true, + errorString: "Error listing nodes using nodeselector", + }, + { + name: "Error listing nodes using nodeselector labels.Everything()", + imageCache: &fledgedv1alpha1.ImageCache{ + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + }, + nodeList: nil, + nodeListError: fmt.Errorf("fake error"), + expectErr: true, + errorString: "Error listing nodes using nodeselector labels.Everything()", + }, + { + name: "NodeSelector did not match any nodes", + imageCache: &fledgedv1alpha1.ImageCache{ + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + NodeSelector: map[string]string{"foo": "bar"}, + }, + }, + }, + }, + nodeList: &corev1.NodeList{}, + nodeListError: nil, + expectErr: true, + errorString: "NodeSelector foo=bar did not match any nodes", + }, + { + name: "Successful validation", + imageCache: &fledgedv1alpha1.ImageCache{ + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + NodeSelector: map[string]string{"foo": "bar"}, + }, + }, + }, + }, + nodeList: &corev1.NodeList{ + Items: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakenode", + Labels: map[string]string{"foo": "bar"}, + }, + }, + }, + }, + nodeListError: nil, + expectErr: false, + errorString: "", + }, + } + + for _, test := range tests { + fakekubeclientset := &fakeclientset.Clientset{} + fakefledgedclientset := &fledgedclientsetfake.Clientset{} + + controller, nodeInformer, _ := newTestController(fakekubeclientset, fakefledgedclientset) + + if test.nodeListError != nil { + //TODO: How to return a fake error from node Lister? + continue + } + + if test.nodeList != nil && len(test.nodeList.Items) > 0 { + for _, node := range test.nodeList.Items { + nodeInformer.Informer().GetIndexer().Add(&node) + } + } + + err := validateCacheSpec(controller, test.imageCache) + if test.expectErr { + if err != nil && strings.HasPrefix(err.Error(), test.errorString) { + } else { + t.Errorf("Test: %s failed", test.name) + } + } else if err != nil { + t.Errorf("Test: %s failed. err received = %s", test.name, err.Error()) + } + } +} diff --git a/cmd/fledged/app/controller_test.go b/cmd/fledged/app/controller_test.go index 8ae27204..8befbcf2 100644 --- a/cmd/fledged/app/controller_test.go +++ b/cmd/fledged/app/controller_test.go @@ -1,1175 +1,1175 @@ -/* -Copyright 2018 The kube-fledged authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "fmt" - "strings" - "testing" - "time" - - fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" - clientset "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned" - fledgedclientsetfake "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/fake" - informers "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions" - fledgedinformers "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions/fledged/v1alpha1" - "github.com/senthilrch/kube-fledged/pkg/images" - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - kubeinformers "k8s.io/client-go/informers" - coreinformers "k8s.io/client-go/informers/core/v1" - "k8s.io/client-go/kubernetes" - fakeclientset "k8s.io/client-go/kubernetes/fake" - core "k8s.io/client-go/testing" -) - -const fledgedNameSpace = "kube-fledged" - -var node = corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"kubernetes.io/hostname": "bar"}, - }, -} - -// noResyncPeriodFunc returns 0 for resyncPeriod in case resyncing is not needed. -func noResyncPeriodFunc() time.Duration { - return 0 -} - -func newTestController(kubeclientset kubernetes.Interface, fledgedclientset clientset.Interface) (*Controller, coreinformers.NodeInformer, fledgedinformers.ImageCacheInformer) { - kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeclientset, noResyncPeriodFunc()) - fledgedInformerFactory := informers.NewSharedInformerFactory(fledgedclientset, noResyncPeriodFunc()) - nodeInformer := kubeInformerFactory.Core().V1().Nodes() - imagecacheInformer := fledgedInformerFactory.Fledged().V1alpha1().ImageCaches() - imageCacheRefreshFrequency := time.Second * 0 - imagePullDeadlineDuration := time.Second * 5 - dockerClientImage := "senthilrch/fledged-docker-client:latest" - imagePullPolicy := "IfNotPresent" - - /* startInformers := true - if startInformers { - stopCh := make(chan struct{}) - defer close(stopCh) - kubeInformerFactory.Start(stopCh) - fledgedInformerFactory.Start(stopCh) - } */ - - controller := NewController(kubeclientset, fledgedclientset, fledgedNameSpace, nodeInformer, imagecacheInformer, - imageCacheRefreshFrequency, imagePullDeadlineDuration, dockerClientImage, imagePullPolicy) - controller.nodesSynced = func() bool { return true } - controller.imageCachesSynced = func() bool { return true } - return controller, nodeInformer, imagecacheInformer -} - -func TestPreFlightChecks(t *testing.T) { - tests := []struct { - name string - jobList *batchv1.JobList - jobListError error - jobDeleteError error - imageCacheList *fledgedv1alpha1.ImageCacheList - imageCacheListError error - imageCacheUpdateError error - expectErr bool - errorString string - }{ - { - name: "#1: No dangling jobs. No imagecaches", - jobList: &batchv1.JobList{Items: []batchv1.Job{}}, - jobListError: nil, - jobDeleteError: nil, - imageCacheList: &fledgedv1alpha1.ImageCacheList{Items: []fledgedv1alpha1.ImageCache{}}, - imageCacheListError: nil, - imageCacheUpdateError: nil, - expectErr: false, - errorString: "", - }, - { - name: "#2: No dangling jobs. No dangling imagecaches", - jobList: &batchv1.JobList{Items: []batchv1.Job{}}, - jobListError: nil, - jobDeleteError: nil, - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, - }, - }, - }, - }, - imageCacheListError: nil, - imageCacheUpdateError: nil, - expectErr: false, - errorString: "", - }, - { - name: "#3: One dangling job. One dangling image cache. Successful list and delete", - //imageCache: nil, - jobList: &batchv1.JobList{ - Items: []batchv1.Job{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - }, - }, - }, - }, - jobListError: nil, - jobDeleteError: nil, - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, - }, - }, - }, - }, - imageCacheListError: nil, - imageCacheUpdateError: nil, - expectErr: false, - errorString: "", - }, - { - name: "#4: Unsuccessful listing of jobs", - jobList: nil, - jobListError: fmt.Errorf("fake error"), - jobDeleteError: nil, - expectErr: true, - errorString: "Internal error occurred: fake error", - }, - { - name: "#5: Unsuccessful listing of imagecaches", - jobList: &batchv1.JobList{Items: []batchv1.Job{}}, - jobListError: nil, - jobDeleteError: nil, - imageCacheList: nil, - imageCacheListError: fmt.Errorf("fake error"), - imageCacheUpdateError: nil, - expectErr: true, - errorString: "Internal error occurred: fake error", - }, - { - name: "#6: One dangling job. Successful list. Unsuccessful delete", - jobList: &batchv1.JobList{ - Items: []batchv1.Job{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - }, - }, - }, - }, - jobListError: nil, - jobDeleteError: fmt.Errorf("fake error"), - expectErr: true, - errorString: "Internal error occurred: fake error", - }, - { - name: "#7: One dangling image cache. Successful list. Unsuccessful delete", - jobList: &batchv1.JobList{Items: []batchv1.Job{}}, - jobListError: nil, - jobDeleteError: nil, - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, - }, - }, - }, - }, - imageCacheListError: nil, - imageCacheUpdateError: fmt.Errorf("fake error"), - expectErr: true, - errorString: "Internal error occurred: fake error", - }, - } - for _, test := range tests { - fakekubeclientset := &fakeclientset.Clientset{} - fakefledgedclientset := &fledgedclientsetfake.Clientset{} - if test.jobListError != nil { - listError := apierrors.NewInternalError(test.jobListError) - fakekubeclientset.AddReactor("list", "jobs", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, listError - }) - } else { - fakekubeclientset.AddReactor("list", "jobs", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, test.jobList, nil - }) - } - if test.jobDeleteError != nil { - deleteError := apierrors.NewInternalError(test.jobDeleteError) - fakekubeclientset.AddReactor("delete", "jobs", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, deleteError - }) - } else { - fakekubeclientset.AddReactor("delete", "jobs", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, nil - }) - } - - if test.imageCacheListError != nil { - listError := apierrors.NewInternalError(test.imageCacheListError) - fakefledgedclientset.AddReactor("list", "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, listError - }) - } else { - fakefledgedclientset.AddReactor("list", "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, test.imageCacheList, nil - }) - } - if test.imageCacheUpdateError != nil { - updateError := apierrors.NewInternalError(test.imageCacheUpdateError) - fakefledgedclientset.AddReactor("update", "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, updateError - }) - } else { - fakefledgedclientset.AddReactor("update", "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, nil - }) - } - - controller, _, _ := newTestController(fakekubeclientset, fakefledgedclientset) - - err := controller.PreFlightChecks() - if test.expectErr { - if !(err != nil && strings.HasPrefix(err.Error(), test.errorString)) { - t.Errorf("Test: %s failed", test.name) - } - } else { - if err != nil { - t.Errorf("Test: %s failed. err received = %s", test.name, err.Error()) - } - } - } - t.Logf("%d tests passed", len(tests)) -} - -func TestRunRefreshWorker(t *testing.T) { - tests := []struct { - name string - imageCacheList *fledgedv1alpha1.ImageCacheList - imageCacheListError error - workqueueItems int - }{ - { - name: "#1: Do not refresh if status is not yet updated", - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - }, - }, - }, - imageCacheListError: nil, - workqueueItems: 0, - }, - { - name: "#2: Do not refresh if image cache is already under processing", - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, - }, - }, - }, - }, - imageCacheListError: nil, - workqueueItems: 0, - }, - { - name: "#3: Do not refresh image cache if cache spec validation failed", - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusFailed, - Reason: fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, - }, - }, - }, - }, - imageCacheListError: nil, - workqueueItems: 0, - }, - { - name: "#4: Do not refresh if image cache has been purged", - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Reason: fledgedv1alpha1.ImageCacheReasonImageCachePurge, - }, - }, - }, - }, - imageCacheListError: nil, - workqueueItems: 0, - }, - { - name: "#5: Successfully queued 1 imagecache for refresh", - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, - }, - }, - }, - }, - imageCacheListError: nil, - workqueueItems: 1, - }, - { - name: "#6: Successfully queued 2 imagecaches for refresh", - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "bar", - Namespace: "kube-fledged", - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusFailed, - }, - }, - }, - }, - imageCacheListError: nil, - workqueueItems: 2, - }, - { - name: "#7: No imagecaches to refresh", - imageCacheList: nil, - imageCacheListError: nil, - workqueueItems: 0, - }, - } - - for _, test := range tests { - if test.workqueueItems > 0 { - //TODO: How to check if workqueue contains the added item? - continue - } - fakekubeclientset := &fakeclientset.Clientset{} - fakefledgedclientset := &fledgedclientsetfake.Clientset{} - - controller, _, imagecacheInformer := newTestController(fakekubeclientset, fakefledgedclientset) - if test.imageCacheList != nil && len(test.imageCacheList.Items) > 0 { - for _, imagecache := range test.imageCacheList.Items { - imagecacheInformer.Informer().GetIndexer().Add(&imagecache) - } - } - controller.runRefreshWorker() - if test.workqueueItems == controller.workqueue.Len() { - } else { - t.Errorf("Test: %s failed: expected %d, actual %d", test.name, test.workqueueItems, controller.workqueue.Len()) - } - } -} - -func TestSyncHandler(t *testing.T) { - type ActionReaction struct { - action string - reaction string - } - now := metav1.Now() - defaultImageCache := fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - } - defaultNodeList := &corev1.NodeList{ - Items: []corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "fakenode", - Labels: map[string]string{"kubernetes.io/hostname": "bar"}, - }, - }, - }, - } - - tests := []struct { - name string - imageCache fledgedv1alpha1.ImageCache - wqKey images.WorkQueueKey - nodeList *corev1.NodeList - expectedActions []ActionReaction - expectErr bool - expectedErrString string - }{ - { - name: "#1: Invalid imagecache resource key", - wqKey: images.WorkQueueKey{ - ObjKey: "foo/bar/car", - }, - expectErr: true, - expectedErrString: "unexpected key format", - }, - { - name: "#2: Create - Invalid imagecache spec (no images specified)", - imageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{}, - }, - }, - }, - }, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheCreate, - }, - expectedActions: []ActionReaction{{action: "update", reaction: ""}}, - expectErr: true, - expectedErrString: "No images specified within image list", - }, - { - name: "#3: Update - Old imagecache pointer is nil", - imageCache: defaultImageCache, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheUpdate, - OldImageCache: nil, - }, - nodeList: defaultNodeList, - expectedActions: []ActionReaction{{action: "update", reaction: ""}}, - expectErr: true, - expectedErrString: "OldImageCacheNotFound", - }, - { - name: "#4: Update - No. of imagelists not equal", - imageCache: defaultImageCache, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheUpdate, - OldImageCache: &fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - { - Images: []string{"bar"}, - }, - }, - }, - }, - }, - nodeList: defaultNodeList, - expectedActions: []ActionReaction{{action: "update", reaction: ""}}, - expectErr: true, - expectedErrString: "CacheSpecValidationFailed", - }, - { - name: "#5: Update - Change in NodeSelectors", - imageCache: defaultImageCache, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheUpdate, - OldImageCache: &fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - NodeSelector: map[string]string{"foo": "bar"}, - }, - }, - }, - }, - }, - nodeList: defaultNodeList, - expectedActions: []ActionReaction{{action: "update", reaction: ""}}, - expectErr: true, - expectedErrString: "CacheSpecValidationFailed", - }, - { - name: "#6: Refresh - Update status to processing", - imageCache: defaultImageCache, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheRefresh, - }, - nodeList: defaultNodeList, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: "fake error"}, - }, - expectErr: true, - expectedErrString: "Internal error occurred: fake error", - }, - { - name: "#7: StatusUpdate - Successful Refresh", - imageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - StartTime: &now, - Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, - Reason: fledgedv1alpha1.ImageCacheReasonImageCacheRefresh, - }, - }, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheStatusUpdate, - Status: &map[string]images.ImageWorkResult{ - "job1": { - Status: images.ImageWorkResultStatusSucceeded, - ImageWorkRequest: images.ImageWorkRequest{ - WorkType: images.ImageCacheRefresh, - Node: &node, - }, - }, - }, - }, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: ""}, - }, - expectErr: false, - expectedErrString: "", - }, - { - name: "#8: Purge - Update status to processing", - imageCache: defaultImageCache, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCachePurge, - }, - nodeList: defaultNodeList, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: "fake error"}, - }, - expectErr: true, - expectedErrString: "Internal error occurred: fake error", - }, - { - name: "#9: Purge - Successful purge", - imageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Reason: fledgedv1alpha1.ImageCacheReasonImageCachePurge, - }, - }, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheRefresh, - }, - nodeList: defaultNodeList, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: ""}, - }, - expectErr: false, - expectedErrString: "", - }, - { - name: "#10: Create - Successfully firing imagepull requests", - imageCache: defaultImageCache, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheCreate, - }, - nodeList: defaultNodeList, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: ""}, - }, - expectErr: false, - expectedErrString: "", - }, - { - name: "#11: Update - Successfully firing imagepull & imagedelete requests", - imageCache: defaultImageCache, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheUpdate, - OldImageCache: &fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo", "bar"}, - }, - }, - }, - }, - }, - nodeList: defaultNodeList, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: ""}, - }, - expectErr: false, - expectedErrString: "", - }, - { - name: "#12: StatusUpdate - ImagesPulledSuccessfully", - imageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - StartTime: &now, - }, - }, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheStatusUpdate, - Status: &map[string]images.ImageWorkResult{ - "job1": { - Status: images.ImageWorkResultStatusSucceeded, - ImageWorkRequest: images.ImageWorkRequest{ - WorkType: images.ImageCacheCreate, - Node: &node, - }, - }, - }, - }, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: ""}, - }, - expectErr: false, - expectedErrString: "", - }, - { - name: "#13: StatusUpdate - ImagesDeletedSuccessfully", - imageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - StartTime: &now, - Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, - Reason: fledgedv1alpha1.ImageCacheReasonImageCachePurge, - }, - }, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheStatusUpdate, - Status: &map[string]images.ImageWorkResult{ - "job1": { - Status: images.ImageWorkResultStatusSucceeded, - ImageWorkRequest: images.ImageWorkRequest{ - WorkType: images.ImageCachePurge, - Node: &node, - }, - }, - }, - }, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: ""}, - }, - expectErr: false, - expectedErrString: "", - }, - { - name: "#14: StatusUpdate - ImagePullFailedForSomeImages", - imageCache: defaultImageCache, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheStatusUpdate, - Status: &map[string]images.ImageWorkResult{ - "job1": { - Status: images.ImageWorkResultStatusFailed, - ImageWorkRequest: images.ImageWorkRequest{ - WorkType: images.ImageCacheCreate, - Node: &node, - }, - }, - }, - }, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: ""}, - }, - expectErr: false, - expectedErrString: "", - }, - { - name: "#15: StatusUpdate - ImageDeleteFailedForSomeImages", - imageCache: defaultImageCache, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheStatusUpdate, - Status: &map[string]images.ImageWorkResult{ - "job1": { - Status: images.ImageWorkResultStatusFailed, - ImageWorkRequest: images.ImageWorkRequest{ - WorkType: images.ImageCachePurge, - Node: &node, - }, - }, - }, - }, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: ""}, - }, - expectErr: false, - expectedErrString: "", - }, - } - - for _, test := range tests { - fakekubeclientset := &fakeclientset.Clientset{} - fakefledgedclientset := &fledgedclientsetfake.Clientset{} - for _, ar := range test.expectedActions { - if ar.reaction != "" { - apiError := apierrors.NewInternalError(fmt.Errorf(ar.reaction)) - fakefledgedclientset.AddReactor(ar.action, "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, apiError - }) - } - fakefledgedclientset.AddReactor(ar.action, "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, &test.imageCache, nil - }) - } - - controller, nodeInformer, imagecacheInformer := newTestController(fakekubeclientset, fakefledgedclientset) - if test.nodeList != nil && len(test.nodeList.Items) > 0 { - for _, node := range test.nodeList.Items { - nodeInformer.Informer().GetIndexer().Add(&node) - } - } - imagecacheInformer.Informer().GetIndexer().Add(&test.imageCache) - err := controller.syncHandler(test.wqKey) - if test.expectErr { - if err == nil { - t.Errorf("Test: %s failed: expectedError=%s, actualError=nil", test.name, test.expectedErrString) - } - if err != nil && !strings.HasPrefix(err.Error(), test.expectedErrString) { - t.Errorf("Test: %s failed: expectedError=%s, actualError=%s", test.name, test.expectedErrString, err.Error()) - } - } else if err != nil { - t.Errorf("Test: %s failed. expectedError=nil, actualError=%s", test.name, err.Error()) - } - } - t.Logf("%d tests passed", len(tests)) -} - -func TestEnqueueImageCache(t *testing.T) { - //now := metav1.Now() - //nowplus5s := metav1.NewTime(time.Now().Add(time.Second * 5)) - defaultImageCache := fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - } - tests := []struct { - name string - workType images.WorkType - oldImageCache fledgedv1alpha1.ImageCache - newImageCache fledgedv1alpha1.ImageCache - expectedResult bool - }{ - { - name: "#1: Create - Imagecache queued successfully", - workType: images.ImageCacheCreate, - newImageCache: defaultImageCache, - expectedResult: true, - }, - { - name: "#2: Create - Imagecache with Status field, so no queueing", - workType: images.ImageCacheCreate, - newImageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, - }, - }, - expectedResult: false, - }, - { - name: "#3: Update - Imagecache purge. Successful queueing", - workType: images.ImageCacheUpdate, - oldImageCache: defaultImageCache, - newImageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - Annotations: map[string]string{imageCachePurgeAnnotationKey: ""}, - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, - }, - }, - expectedResult: true, - }, - { - name: "#4: Update - No change in Spec. Unsuccessful queueing", - workType: images.ImageCacheUpdate, - oldImageCache: defaultImageCache, - newImageCache: defaultImageCache, - expectedResult: false, - }, - { - name: "#5: Update - Status processing. Unsuccessful queueing", - workType: images.ImageCacheUpdate, - oldImageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, - }, - }, - expectedResult: false, - }, - { - name: "#6: Update - Successful queueing", - workType: images.ImageCacheUpdate, - oldImageCache: defaultImageCache, - newImageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo", "bar"}, - }, - }, - }, - }, - expectedResult: true, - }, - { - name: "#7: Delete - Unsuccessful queueing", - workType: images.ImageCacheDelete, - expectedResult: false, - }, - { - name: "#8: Refresh - Successful queueing", - workType: images.ImageCacheRefresh, - oldImageCache: defaultImageCache, - expectedResult: true, - }, - { - name: "#9: Update - CacheSpec restoration", - workType: images.ImageCacheUpdate, - oldImageCache: defaultImageCache, - newImageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - Annotations: map[string]string{ - fledgedCacheSpecValidationKey: "failed", - }, - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo", "bar"}, - }, - }, - }, - }, - expectedResult: false, - }, - { - name: "#10: Update - Imagecache refresh. Successful queueing", - workType: images.ImageCacheUpdate, - oldImageCache: defaultImageCache, - newImageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - Annotations: map[string]string{imageCacheRefreshAnnotationKey: ""}, - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, - }, - }, - expectedResult: true, - }, - } - - for _, test := range tests { - fakekubeclientset := &fakeclientset.Clientset{} - fakefledgedclientset := &fledgedclientsetfake.Clientset{} - controller, _, _ := newTestController(fakekubeclientset, fakefledgedclientset) - result := controller.enqueueImageCache(test.workType, &test.oldImageCache, &test.newImageCache) - if result != test.expectedResult { - t.Errorf("Test %s failed: expected=%t, actual=%t", test.name, test.expectedResult, result) - } - } -} - -func TestProcessNextWorkItem(t *testing.T) { - type ActionReaction struct { - action string - reaction string - } - defaultImageCache := fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - } - - tests := []struct { - name string - imageCache fledgedv1alpha1.ImageCache - wqKey images.WorkQueueKey - expectedActions []ActionReaction - expectErr bool - expectedErrString string - }{ - { - name: "#1: StatusUpdate - ImageDeleteFailedForSomeImages", - imageCache: defaultImageCache, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheStatusUpdate, - Status: &map[string]images.ImageWorkResult{ - "job1": { - Status: images.ImageWorkResultStatusFailed, - ImageWorkRequest: images.ImageWorkRequest{ - WorkType: images.ImageCachePurge, - Node: &node, - }, - }, - }, - }, - expectedActions: []ActionReaction{ - {action: "get", reaction: ""}, - {action: "update", reaction: ""}, - }, - expectErr: false, - expectedErrString: "", - }, - { - name: "#2: Create - Invalid imagecache spec (no images specified)", - imageCache: fledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{}, - }, - }, - }, - }, - wqKey: images.WorkQueueKey{ - ObjKey: "kube-fledged/foo", - WorkType: images.ImageCacheCreate, - }, - expectedActions: []ActionReaction{{action: "update", reaction: ""}}, - expectErr: false, - expectedErrString: "No images specified within image list", - }, - { - name: "#3: Unexpected type in workqueue", - expectErr: false, - expectedErrString: "Unexpected type in workqueue", - }, - } - - for _, test := range tests { - fakekubeclientset := &fakeclientset.Clientset{} - fakefledgedclientset := &fledgedclientsetfake.Clientset{} - for _, ar := range test.expectedActions { - if ar.reaction != "" { - apiError := apierrors.NewInternalError(fmt.Errorf(ar.reaction)) - fakefledgedclientset.AddReactor(ar.action, "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, apiError - }) - } - fakefledgedclientset.AddReactor(ar.action, "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, &test.imageCache, nil - }) - } - - controller, _, imagecacheInformer := newTestController(fakekubeclientset, fakefledgedclientset) - imagecacheInformer.Informer().GetIndexer().Add(&test.imageCache) - if test.expectedErrString == "Unexpected type in workqueue" { - controller.workqueue.Add(struct{}{}) - } - controller.workqueue.Add(test.wqKey) - controller.processNextWorkItem() - var err error - if test.expectErr { - if err == nil { - t.Errorf("Test: %s failed: expectedError=%s, actualError=nil", test.name, test.expectedErrString) - } - if err != nil && !strings.HasPrefix(err.Error(), test.expectedErrString) { - t.Errorf("Test: %s failed: expectedError=%s, actualError=%s", test.name, test.expectedErrString, err.Error()) - } - } else if err != nil { - t.Errorf("Test: %s failed. expectedError=nil, actualError=%s", test.name, err.Error()) - } - } - t.Logf("%d tests passed", len(tests)) -} +/* +Copyright 2018 The kube-fledged authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + "strings" + "testing" + "time" + + fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + clientset "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned" + fledgedclientsetfake "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/fake" + informers "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions" + fledgedinformers "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions/fledged/v1alpha1" + "github.com/senthilrch/kube-fledged/pkg/images" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubeinformers "k8s.io/client-go/informers" + coreinformers "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/kubernetes" + fakeclientset "k8s.io/client-go/kubernetes/fake" + core "k8s.io/client-go/testing" +) + +const fledgedNameSpace = "kube-fledged" + +var node = corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"kubernetes.io/hostname": "bar"}, + }, +} + +// noResyncPeriodFunc returns 0 for resyncPeriod in case resyncing is not needed. +func noResyncPeriodFunc() time.Duration { + return 0 +} + +func newTestController(kubeclientset kubernetes.Interface, fledgedclientset clientset.Interface) (*Controller, coreinformers.NodeInformer, fledgedinformers.ImageCacheInformer) { + kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeclientset, noResyncPeriodFunc()) + fledgedInformerFactory := informers.NewSharedInformerFactory(fledgedclientset, noResyncPeriodFunc()) + nodeInformer := kubeInformerFactory.Core().V1().Nodes() + imagecacheInformer := fledgedInformerFactory.Fledged().V1alpha1().ImageCaches() + imageCacheRefreshFrequency := time.Second * 0 + imagePullDeadlineDuration := time.Second * 5 + dockerClientImage := "senthilrch/fledged-docker-client:latest" + imagePullPolicy := "IfNotPresent" + + /* startInformers := true + if startInformers { + stopCh := make(chan struct{}) + defer close(stopCh) + kubeInformerFactory.Start(stopCh) + fledgedInformerFactory.Start(stopCh) + } */ + + controller := NewController(kubeclientset, fledgedclientset, fledgedNameSpace, nodeInformer, imagecacheInformer, + imageCacheRefreshFrequency, imagePullDeadlineDuration, dockerClientImage, imagePullPolicy) + controller.nodesSynced = func() bool { return true } + controller.imageCachesSynced = func() bool { return true } + return controller, nodeInformer, imagecacheInformer +} + +func TestPreFlightChecks(t *testing.T) { + tests := []struct { + name string + jobList *batchv1.JobList + jobListError error + jobDeleteError error + imageCacheList *fledgedv1alpha1.ImageCacheList + imageCacheListError error + imageCacheUpdateError error + expectErr bool + errorString string + }{ + { + name: "#1: No dangling jobs. No imagecaches", + jobList: &batchv1.JobList{Items: []batchv1.Job{}}, + jobListError: nil, + jobDeleteError: nil, + imageCacheList: &fledgedv1alpha1.ImageCacheList{Items: []fledgedv1alpha1.ImageCache{}}, + imageCacheListError: nil, + imageCacheUpdateError: nil, + expectErr: false, + errorString: "", + }, + { + name: "#2: No dangling jobs. No dangling imagecaches", + jobList: &batchv1.JobList{Items: []batchv1.Job{}}, + jobListError: nil, + jobDeleteError: nil, + imageCacheList: &fledgedv1alpha1.ImageCacheList{ + Items: []fledgedv1alpha1.ImageCache{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, + }, + }, + }, + }, + imageCacheListError: nil, + imageCacheUpdateError: nil, + expectErr: false, + errorString: "", + }, + { + name: "#3: One dangling job. One dangling image cache. Successful list and delete", + //imageCache: nil, + jobList: &batchv1.JobList{ + Items: []batchv1.Job{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + }, + }, + }, + jobListError: nil, + jobDeleteError: nil, + imageCacheList: &fledgedv1alpha1.ImageCacheList{ + Items: []fledgedv1alpha1.ImageCache{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, + }, + }, + }, + }, + imageCacheListError: nil, + imageCacheUpdateError: nil, + expectErr: false, + errorString: "", + }, + { + name: "#4: Unsuccessful listing of jobs", + jobList: nil, + jobListError: fmt.Errorf("fake error"), + jobDeleteError: nil, + expectErr: true, + errorString: "Internal error occurred: fake error", + }, + { + name: "#5: Unsuccessful listing of imagecaches", + jobList: &batchv1.JobList{Items: []batchv1.Job{}}, + jobListError: nil, + jobDeleteError: nil, + imageCacheList: nil, + imageCacheListError: fmt.Errorf("fake error"), + imageCacheUpdateError: nil, + expectErr: true, + errorString: "Internal error occurred: fake error", + }, + { + name: "#6: One dangling job. Successful list. Unsuccessful delete", + jobList: &batchv1.JobList{ + Items: []batchv1.Job{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + }, + }, + }, + jobListError: nil, + jobDeleteError: fmt.Errorf("fake error"), + expectErr: true, + errorString: "Internal error occurred: fake error", + }, + { + name: "#7: One dangling image cache. Successful list. Unsuccessful delete", + jobList: &batchv1.JobList{Items: []batchv1.Job{}}, + jobListError: nil, + jobDeleteError: nil, + imageCacheList: &fledgedv1alpha1.ImageCacheList{ + Items: []fledgedv1alpha1.ImageCache{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, + }, + }, + }, + }, + imageCacheListError: nil, + imageCacheUpdateError: fmt.Errorf("fake error"), + expectErr: true, + errorString: "Internal error occurred: fake error", + }, + } + for _, test := range tests { + fakekubeclientset := &fakeclientset.Clientset{} + fakefledgedclientset := &fledgedclientsetfake.Clientset{} + if test.jobListError != nil { + listError := apierrors.NewInternalError(test.jobListError) + fakekubeclientset.AddReactor("list", "jobs", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, listError + }) + } else { + fakekubeclientset.AddReactor("list", "jobs", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, test.jobList, nil + }) + } + if test.jobDeleteError != nil { + deleteError := apierrors.NewInternalError(test.jobDeleteError) + fakekubeclientset.AddReactor("delete", "jobs", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, deleteError + }) + } else { + fakekubeclientset.AddReactor("delete", "jobs", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, nil + }) + } + + if test.imageCacheListError != nil { + listError := apierrors.NewInternalError(test.imageCacheListError) + fakefledgedclientset.AddReactor("list", "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, listError + }) + } else { + fakefledgedclientset.AddReactor("list", "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, test.imageCacheList, nil + }) + } + if test.imageCacheUpdateError != nil { + updateError := apierrors.NewInternalError(test.imageCacheUpdateError) + fakefledgedclientset.AddReactor("update", "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, updateError + }) + } else { + fakefledgedclientset.AddReactor("update", "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, nil + }) + } + + controller, _, _ := newTestController(fakekubeclientset, fakefledgedclientset) + + err := controller.PreFlightChecks() + if test.expectErr { + if !(err != nil && strings.HasPrefix(err.Error(), test.errorString)) { + t.Errorf("Test: %s failed", test.name) + } + } else { + if err != nil { + t.Errorf("Test: %s failed. err received = %s", test.name, err.Error()) + } + } + } + t.Logf("%d tests passed", len(tests)) +} + +func TestRunRefreshWorker(t *testing.T) { + tests := []struct { + name string + imageCacheList *fledgedv1alpha1.ImageCacheList + imageCacheListError error + workqueueItems int + }{ + { + name: "#1: Do not refresh if status is not yet updated", + imageCacheList: &fledgedv1alpha1.ImageCacheList{ + Items: []fledgedv1alpha1.ImageCache{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + }, + }, + }, + imageCacheListError: nil, + workqueueItems: 0, + }, + { + name: "#2: Do not refresh if image cache is already under processing", + imageCacheList: &fledgedv1alpha1.ImageCacheList{ + Items: []fledgedv1alpha1.ImageCache{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, + }, + }, + }, + }, + imageCacheListError: nil, + workqueueItems: 0, + }, + { + name: "#3: Do not refresh image cache if cache spec validation failed", + imageCacheList: &fledgedv1alpha1.ImageCacheList{ + Items: []fledgedv1alpha1.ImageCache{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusFailed, + Reason: fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, + }, + }, + }, + }, + imageCacheListError: nil, + workqueueItems: 0, + }, + { + name: "#4: Do not refresh if image cache has been purged", + imageCacheList: &fledgedv1alpha1.ImageCacheList{ + Items: []fledgedv1alpha1.ImageCache{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Reason: fledgedv1alpha1.ImageCacheReasonImageCachePurge, + }, + }, + }, + }, + imageCacheListError: nil, + workqueueItems: 0, + }, + { + name: "#5: Successfully queued 1 imagecache for refresh", + imageCacheList: &fledgedv1alpha1.ImageCacheList{ + Items: []fledgedv1alpha1.ImageCache{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, + }, + }, + }, + }, + imageCacheListError: nil, + workqueueItems: 1, + }, + { + name: "#6: Successfully queued 2 imagecaches for refresh", + imageCacheList: &fledgedv1alpha1.ImageCacheList{ + Items: []fledgedv1alpha1.ImageCache{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Namespace: "kube-fledged", + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusFailed, + }, + }, + }, + }, + imageCacheListError: nil, + workqueueItems: 2, + }, + { + name: "#7: No imagecaches to refresh", + imageCacheList: nil, + imageCacheListError: nil, + workqueueItems: 0, + }, + } + + for _, test := range tests { + if test.workqueueItems > 0 { + //TODO: How to check if workqueue contains the added item? + continue + } + fakekubeclientset := &fakeclientset.Clientset{} + fakefledgedclientset := &fledgedclientsetfake.Clientset{} + + controller, _, imagecacheInformer := newTestController(fakekubeclientset, fakefledgedclientset) + if test.imageCacheList != nil && len(test.imageCacheList.Items) > 0 { + for _, imagecache := range test.imageCacheList.Items { + imagecacheInformer.Informer().GetIndexer().Add(&imagecache) + } + } + controller.runRefreshWorker() + if test.workqueueItems == controller.workqueue.Len() { + } else { + t.Errorf("Test: %s failed: expected %d, actual %d", test.name, test.workqueueItems, controller.workqueue.Len()) + } + } +} + +func TestSyncHandler(t *testing.T) { + type ActionReaction struct { + action string + reaction string + } + now := metav1.Now() + defaultImageCache := fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + } + defaultNodeList := &corev1.NodeList{ + Items: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "fakenode", + Labels: map[string]string{"kubernetes.io/hostname": "bar"}, + }, + }, + }, + } + + tests := []struct { + name string + imageCache fledgedv1alpha1.ImageCache + wqKey images.WorkQueueKey + nodeList *corev1.NodeList + expectedActions []ActionReaction + expectErr bool + expectedErrString string + }{ + { + name: "#1: Invalid imagecache resource key", + wqKey: images.WorkQueueKey{ + ObjKey: "foo/bar/car", + }, + expectErr: true, + expectedErrString: "unexpected key format", + }, + { + name: "#2: Create - Invalid imagecache spec (no images specified)", + imageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{}, + }, + }, + }, + }, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheCreate, + }, + expectedActions: []ActionReaction{{action: "update", reaction: ""}}, + expectErr: true, + expectedErrString: "No images specified within image list", + }, + { + name: "#3: Update - Old imagecache pointer is nil", + imageCache: defaultImageCache, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheUpdate, + OldImageCache: nil, + }, + nodeList: defaultNodeList, + expectedActions: []ActionReaction{{action: "update", reaction: ""}}, + expectErr: true, + expectedErrString: "OldImageCacheNotFound", + }, + { + name: "#4: Update - No. of imagelists not equal", + imageCache: defaultImageCache, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheUpdate, + OldImageCache: &fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + { + Images: []string{"bar"}, + }, + }, + }, + }, + }, + nodeList: defaultNodeList, + expectedActions: []ActionReaction{{action: "update", reaction: ""}}, + expectErr: true, + expectedErrString: "CacheSpecValidationFailed", + }, + { + name: "#5: Update - Change in NodeSelectors", + imageCache: defaultImageCache, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheUpdate, + OldImageCache: &fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + NodeSelector: map[string]string{"foo": "bar"}, + }, + }, + }, + }, + }, + nodeList: defaultNodeList, + expectedActions: []ActionReaction{{action: "update", reaction: ""}}, + expectErr: true, + expectedErrString: "CacheSpecValidationFailed", + }, + { + name: "#6: Refresh - Update status to processing", + imageCache: defaultImageCache, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheRefresh, + }, + nodeList: defaultNodeList, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: "fake error"}, + }, + expectErr: true, + expectedErrString: "Internal error occurred: fake error", + }, + { + name: "#7: StatusUpdate - Successful Refresh", + imageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + StartTime: &now, + Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, + Reason: fledgedv1alpha1.ImageCacheReasonImageCacheRefresh, + }, + }, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheStatusUpdate, + Status: &map[string]images.ImageWorkResult{ + "job1": { + Status: images.ImageWorkResultStatusSucceeded, + ImageWorkRequest: images.ImageWorkRequest{ + WorkType: images.ImageCacheRefresh, + Node: &node, + }, + }, + }, + }, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: ""}, + }, + expectErr: false, + expectedErrString: "", + }, + { + name: "#8: Purge - Update status to processing", + imageCache: defaultImageCache, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCachePurge, + }, + nodeList: defaultNodeList, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: "fake error"}, + }, + expectErr: true, + expectedErrString: "Internal error occurred: fake error", + }, + { + name: "#9: Purge - Successful purge", + imageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Reason: fledgedv1alpha1.ImageCacheReasonImageCachePurge, + }, + }, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheRefresh, + }, + nodeList: defaultNodeList, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: ""}, + }, + expectErr: false, + expectedErrString: "", + }, + { + name: "#10: Create - Successfully firing imagepull requests", + imageCache: defaultImageCache, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheCreate, + }, + nodeList: defaultNodeList, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: ""}, + }, + expectErr: false, + expectedErrString: "", + }, + { + name: "#11: Update - Successfully firing imagepull & imagedelete requests", + imageCache: defaultImageCache, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheUpdate, + OldImageCache: &fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo", "bar"}, + }, + }, + }, + }, + }, + nodeList: defaultNodeList, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: ""}, + }, + expectErr: false, + expectedErrString: "", + }, + { + name: "#12: StatusUpdate - ImagesPulledSuccessfully", + imageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + StartTime: &now, + }, + }, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheStatusUpdate, + Status: &map[string]images.ImageWorkResult{ + "job1": { + Status: images.ImageWorkResultStatusSucceeded, + ImageWorkRequest: images.ImageWorkRequest{ + WorkType: images.ImageCacheCreate, + Node: &node, + }, + }, + }, + }, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: ""}, + }, + expectErr: false, + expectedErrString: "", + }, + { + name: "#13: StatusUpdate - ImagesDeletedSuccessfully", + imageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + StartTime: &now, + Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, + Reason: fledgedv1alpha1.ImageCacheReasonImageCachePurge, + }, + }, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheStatusUpdate, + Status: &map[string]images.ImageWorkResult{ + "job1": { + Status: images.ImageWorkResultStatusSucceeded, + ImageWorkRequest: images.ImageWorkRequest{ + WorkType: images.ImageCachePurge, + Node: &node, + }, + }, + }, + }, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: ""}, + }, + expectErr: false, + expectedErrString: "", + }, + { + name: "#14: StatusUpdate - ImagePullFailedForSomeImages", + imageCache: defaultImageCache, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheStatusUpdate, + Status: &map[string]images.ImageWorkResult{ + "job1": { + Status: images.ImageWorkResultStatusFailed, + ImageWorkRequest: images.ImageWorkRequest{ + WorkType: images.ImageCacheCreate, + Node: &node, + }, + }, + }, + }, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: ""}, + }, + expectErr: false, + expectedErrString: "", + }, + { + name: "#15: StatusUpdate - ImageDeleteFailedForSomeImages", + imageCache: defaultImageCache, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheStatusUpdate, + Status: &map[string]images.ImageWorkResult{ + "job1": { + Status: images.ImageWorkResultStatusFailed, + ImageWorkRequest: images.ImageWorkRequest{ + WorkType: images.ImageCachePurge, + Node: &node, + }, + }, + }, + }, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: ""}, + }, + expectErr: false, + expectedErrString: "", + }, + } + + for _, test := range tests { + fakekubeclientset := &fakeclientset.Clientset{} + fakefledgedclientset := &fledgedclientsetfake.Clientset{} + for _, ar := range test.expectedActions { + if ar.reaction != "" { + apiError := apierrors.NewInternalError(fmt.Errorf(ar.reaction)) + fakefledgedclientset.AddReactor(ar.action, "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, apiError + }) + } + fakefledgedclientset.AddReactor(ar.action, "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, &test.imageCache, nil + }) + } + + controller, nodeInformer, imagecacheInformer := newTestController(fakekubeclientset, fakefledgedclientset) + if test.nodeList != nil && len(test.nodeList.Items) > 0 { + for _, node := range test.nodeList.Items { + nodeInformer.Informer().GetIndexer().Add(&node) + } + } + imagecacheInformer.Informer().GetIndexer().Add(&test.imageCache) + err := controller.syncHandler(test.wqKey) + if test.expectErr { + if err == nil { + t.Errorf("Test: %s failed: expectedError=%s, actualError=nil", test.name, test.expectedErrString) + } + if err != nil && !strings.HasPrefix(err.Error(), test.expectedErrString) { + t.Errorf("Test: %s failed: expectedError=%s, actualError=%s", test.name, test.expectedErrString, err.Error()) + } + } else if err != nil { + t.Errorf("Test: %s failed. expectedError=nil, actualError=%s", test.name, err.Error()) + } + } + t.Logf("%d tests passed", len(tests)) +} + +func TestEnqueueImageCache(t *testing.T) { + //now := metav1.Now() + //nowplus5s := metav1.NewTime(time.Now().Add(time.Second * 5)) + defaultImageCache := fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + } + tests := []struct { + name string + workType images.WorkType + oldImageCache fledgedv1alpha1.ImageCache + newImageCache fledgedv1alpha1.ImageCache + expectedResult bool + }{ + { + name: "#1: Create - Imagecache queued successfully", + workType: images.ImageCacheCreate, + newImageCache: defaultImageCache, + expectedResult: true, + }, + { + name: "#2: Create - Imagecache with Status field, so no queueing", + workType: images.ImageCacheCreate, + newImageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, + }, + }, + expectedResult: false, + }, + { + name: "#3: Update - Imagecache purge. Successful queueing", + workType: images.ImageCacheUpdate, + oldImageCache: defaultImageCache, + newImageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + Annotations: map[string]string{imageCachePurgeAnnotationKey: ""}, + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, + }, + }, + expectedResult: true, + }, + { + name: "#4: Update - No change in Spec. Unsuccessful queueing", + workType: images.ImageCacheUpdate, + oldImageCache: defaultImageCache, + newImageCache: defaultImageCache, + expectedResult: false, + }, + { + name: "#5: Update - Status processing. Unsuccessful queueing", + workType: images.ImageCacheUpdate, + oldImageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, + }, + }, + expectedResult: false, + }, + { + name: "#6: Update - Successful queueing", + workType: images.ImageCacheUpdate, + oldImageCache: defaultImageCache, + newImageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo", "bar"}, + }, + }, + }, + }, + expectedResult: true, + }, + { + name: "#7: Delete - Unsuccessful queueing", + workType: images.ImageCacheDelete, + expectedResult: false, + }, + { + name: "#8: Refresh - Successful queueing", + workType: images.ImageCacheRefresh, + oldImageCache: defaultImageCache, + expectedResult: true, + }, + { + name: "#9: Update - CacheSpec restoration", + workType: images.ImageCacheUpdate, + oldImageCache: defaultImageCache, + newImageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + Annotations: map[string]string{ + fledgedCacheSpecValidationKey: "failed", + }, + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo", "bar"}, + }, + }, + }, + }, + expectedResult: false, + }, + { + name: "#10: Update - Imagecache refresh. Successful queueing", + workType: images.ImageCacheUpdate, + oldImageCache: defaultImageCache, + newImageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + Annotations: map[string]string{imageCacheRefreshAnnotationKey: ""}, + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + Status: fledgedv1alpha1.ImageCacheStatus{ + Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, + }, + }, + expectedResult: true, + }, + } + + for _, test := range tests { + fakekubeclientset := &fakeclientset.Clientset{} + fakefledgedclientset := &fledgedclientsetfake.Clientset{} + controller, _, _ := newTestController(fakekubeclientset, fakefledgedclientset) + result := controller.enqueueImageCache(test.workType, &test.oldImageCache, &test.newImageCache) + if result != test.expectedResult { + t.Errorf("Test %s failed: expected=%t, actual=%t", test.name, test.expectedResult, result) + } + } +} + +func TestProcessNextWorkItem(t *testing.T) { + type ActionReaction struct { + action string + reaction string + } + defaultImageCache := fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo"}, + }, + }, + }, + } + + tests := []struct { + name string + imageCache fledgedv1alpha1.ImageCache + wqKey images.WorkQueueKey + expectedActions []ActionReaction + expectErr bool + expectedErrString string + }{ + { + name: "#1: StatusUpdate - ImageDeleteFailedForSomeImages", + imageCache: defaultImageCache, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheStatusUpdate, + Status: &map[string]images.ImageWorkResult{ + "job1": { + Status: images.ImageWorkResultStatusFailed, + ImageWorkRequest: images.ImageWorkRequest{ + WorkType: images.ImageCachePurge, + Node: &node, + }, + }, + }, + }, + expectedActions: []ActionReaction{ + {action: "get", reaction: ""}, + {action: "update", reaction: ""}, + }, + expectErr: false, + expectedErrString: "", + }, + { + name: "#2: Create - Invalid imagecache spec (no images specified)", + imageCache: fledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + }, + Spec: fledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + { + Images: []string{}, + }, + }, + }, + }, + wqKey: images.WorkQueueKey{ + ObjKey: "kube-fledged/foo", + WorkType: images.ImageCacheCreate, + }, + expectedActions: []ActionReaction{{action: "update", reaction: ""}}, + expectErr: false, + expectedErrString: "No images specified within image list", + }, + { + name: "#3: Unexpected type in workqueue", + expectErr: false, + expectedErrString: "Unexpected type in workqueue", + }, + } + + for _, test := range tests { + fakekubeclientset := &fakeclientset.Clientset{} + fakefledgedclientset := &fledgedclientsetfake.Clientset{} + for _, ar := range test.expectedActions { + if ar.reaction != "" { + apiError := apierrors.NewInternalError(fmt.Errorf(ar.reaction)) + fakefledgedclientset.AddReactor(ar.action, "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, apiError + }) + } + fakefledgedclientset.AddReactor(ar.action, "imagecaches", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, &test.imageCache, nil + }) + } + + controller, _, imagecacheInformer := newTestController(fakekubeclientset, fakefledgedclientset) + imagecacheInformer.Informer().GetIndexer().Add(&test.imageCache) + if test.expectedErrString == "Unexpected type in workqueue" { + controller.workqueue.Add(struct{}{}) + } + controller.workqueue.Add(test.wqKey) + controller.processNextWorkItem() + var err error + if test.expectErr { + if err == nil { + t.Errorf("Test: %s failed: expectedError=%s, actualError=nil", test.name, test.expectedErrString) + } + if err != nil && !strings.HasPrefix(err.Error(), test.expectedErrString) { + t.Errorf("Test: %s failed: expectedError=%s, actualError=%s", test.name, test.expectedErrString, err.Error()) + } + } else if err != nil { + t.Errorf("Test: %s failed. expectedError=nil, actualError=%s", test.name, err.Error()) + } + } + t.Logf("%d tests passed", len(tests)) +} From b4e3a0709fc1cb45d8c5f3040bf73531b9c783a3 Mon Sep 17 00:00:00 2001 From: bhuvanessr Date: Wed, 27 May 2020 20:05:41 +0530 Subject: [PATCH 10/21] changes to makefile and dockerfiles --- Makefile | 24 +++++++++++++++++------- build/Dockerfile.fledged | 2 +- build/Dockerfile.webhook_server | 2 +- build/Dockerfile.webhook_server_dev | 23 +++++++++++++++++++++++ 4 files changed, 42 insertions(+), 9 deletions(-) create mode 100644 build/Dockerfile.webhook_server_dev diff --git a/Makefile b/Makefile index 7e85d067..b31f27a3 100644 --- a/Makefile +++ b/Makefile @@ -30,8 +30,8 @@ ifndef FLEDGED_IMAGE_REPO FLEDGED_IMAGE_REPO=docker.io/senthilrch/fledged endif -ifndef WEBHOOK_SERVER_IMAGE_REPO - WEBHOOK_SERVER_IMAGE_REPO=docker.io/senthilrch/fledged-webhook-server +ifndef FLEDGED_WEBHOOK_SERVER_IMAGE_REPO + FLEDGED_WEBHOOK_SERVER_IMAGE_REPO=docker.io/senthilrch/fledged-webhook-server endif ifndef FLEDGED_DOCKER_CLIENT_IMAGE_REPO @@ -103,7 +103,7 @@ clean-fledged: clean-webhook-server: -rm -f build/webhook-server - -docker image rm ${WEBHOOK_SERVER_IMAGE_REPO}:${RELEASE_VERSION} + -docker image rm ${FLEDGED_WEBHOOK_SERVER_IMAGE_REPO}:${RELEASE_VERSION} -docker image rm `docker image ls -f dangling=true -q` clean-client: @@ -122,13 +122,21 @@ fledged-amd64: clean-fledged fledged-dev: fledged-amd64 docker push ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION} +webhook-server-amd64: clean-webhook-server + CGO_ENABLED=0 go build -o build/fledged-webhook-server -ldflags '-s -w -extldflags "-static"' cmd/webhook-server/main.go && \ + cd build && docker build -t ${FLEDGED_WEBHOOK_SERVER_IMAGE_REPO}:${RELEASE_VERSION} -f Dockerfile.webhook_server_dev \ + --build-arg ALPINE_VERSION=${ALPINE_VERSION} . + +webhook-server-dev: webhook-server-amd64 + docker push ${FLEDGED_WEBHOOK_SERVER_IMAGE_REPO}:${RELEASE_VERSION} + fledged-image: clean-fledged cd build && docker buildx build --platform=${TARGET_PLATFORMS} -t ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION} \ -f Dockerfile.fledged ${HTTP_PROXY_CONFIG} ${HTTPS_PROXY_CONFIG} --build-arg GIT_BRANCH=${GIT_BRANCH} \ --build-arg GOLANG_VERSION=${GOLANG_VERSION} --build-arg ALPINE_VERSION=${ALPINE_VERSION} --progress=plain ${BUILD_OUTPUT} . webhook-server-image: clean-webhook-server - cd build && docker buildx build --platform=${TARGET_PLATFORMS} -t ${WEBHOOK_SERVER_IMAGE_REPO}:${RELEASE_VERSION} \ + cd build && docker buildx build --platform=${TARGET_PLATFORMS} -t ${FLEDGED_WEBHOOK_SERVER_IMAGE_REPO}:${RELEASE_VERSION} \ -f Dockerfile.webhook_server ${HTTP_PROXY_CONFIG} ${HTTPS_PROXY_CONFIG} --build-arg GIT_BRANCH=${GIT_BRANCH} \ --build-arg GOLANG_VERSION=${GOLANG_VERSION} --build-arg ALPINE_VERSION=${ALPINE_VERSION} --progress=plain ${BUILD_OUTPUT} . @@ -145,10 +153,11 @@ operator-image: clean-operator push-images: -docker push ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION} + -docker push ${FLEDGED_WEBHOOK_SERVER_IMAGE_REPO}:${RELEASE_VERSION} -docker push ${FLEDGED_DOCKER_CLIENT_IMAGE_REPO}:${RELEASE_VERSION} -docker push ${OPERATOR_IMAGE_REPO}:${RELEASE_VERSION} -release: install-buildx fledged-image client-image operator-image +release: install-buildx fledged-image webhook-server-image client-image operator-image install-buildx: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes @@ -162,12 +171,13 @@ test: bash hack/run-unit-tests.sh deploy-using-yaml: + -kubectl apply -f deploy/kubefledged-namespace.yaml kubectl apply -f deploy/kubefledged-crd.yaml && \ - kubectl apply -f deploy/kubefledged-namespace.yaml && \ kubectl apply -f deploy/kubefledged-serviceaccount.yaml && \ kubectl apply -f deploy/kubefledged-clusterrole.yaml && \ kubectl apply -f deploy/kubefledged-clusterrolebinding.yaml && \ - kubectl apply -f deploy/kubefledged-deployment.yaml + kubectl apply -f deploy/kubefledged-deployment-fledged-controller.yaml && \ + kubectl apply -f deploy/kubefledged-deployment-webhook-server.yaml deploy-using-operator: # Deploy the operator to a separate namespace called "operators" diff --git a/build/Dockerfile.fledged b/build/Dockerfile.fledged index 7558501d..3748e3fd 100644 --- a/build/Dockerfile.fledged +++ b/build/Dockerfile.fledged @@ -19,7 +19,7 @@ FROM golang:$GOLANG_VERSION AS builder LABEL stage=builder ARG GIT_BRANCH RUN mkdir -p /go/src/github.com/senthilrch && \ - git clone --depth=1 --single-branch --branch=$GIT_BRANCH https://github.com/senthilrch/kube-fledged /go/src/github.com/senthilrch/kube-fledged && \ + git clone --depth=1 --single-branch --branch=$GIT_BRANCH https://github.com/senthilrch/kube-fledged.git /go/src/github.com/senthilrch/kube-fledged && \ cd /go/src/github.com/senthilrch/kube-fledged && \ CGO_ENABLED=0 go build -o build/fledged -ldflags '-s -w -extldflags "-static"' cmd/fledged/main.go diff --git a/build/Dockerfile.webhook_server b/build/Dockerfile.webhook_server index 0446fffc..9cf8ea49 100644 --- a/build/Dockerfile.webhook_server +++ b/build/Dockerfile.webhook_server @@ -19,7 +19,7 @@ FROM golang:$GOLANG_VERSION AS builder LABEL stage=builder ARG GIT_BRANCH RUN mkdir -p /go/src/github.com/senthilrch && \ - git clone --depth=1 --single-branch --branch=$GIT_BRANCH https://github.com/senthilrch/kube-fledged /go/src/github.com/senthilrch/kube-fledged && \ + git clone --depth=1 --single-branch --branch=$GIT_BRANCH https://github.com/senthilrch/kube-fledged.git /go/src/github.com/senthilrch/kube-fledged && \ cd /go/src/github.com/senthilrch/kube-fledged && \ CGO_ENABLED=0 go build -o build/fledged-webhook-server -ldflags '-s -w -extldflags "-static"' cmd/webhook-server/main.go diff --git a/build/Dockerfile.webhook_server_dev b/build/Dockerfile.webhook_server_dev new file mode 100644 index 00000000..08def573 --- /dev/null +++ b/build/Dockerfile.webhook_server_dev @@ -0,0 +1,23 @@ +# Copyright 2018 The kube-fledged authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG ALPINE_VERSION + +FROM alpine:$ALPINE_VERSION +LABEL maintainer="senthilrch " + +COPY fledged-webhook-server /opt/bin/fledged-webhook-server +RUN chmod 755 /opt/bin/fledged-webhook-server + +ENTRYPOINT ["/opt/bin/fledged-webhook-server"] From d6f0a4f41e3afb83897880d275890d14b2c6c1ca Mon Sep 17 00:00:00 2001 From: bhuvanessr Date: Wed, 27 May 2020 22:36:30 +0530 Subject: [PATCH 11/21] updated yaml manifest files --- Makefile | 8 +- deploy/create-cert-key-secret.sh | 131 ++++++++++++++++++ ...kubefledged-deployment-webhook-server.yaml | 5 +- .../kubefledged-service-webhook-server.yaml | 14 ++ deploy/kubefledged-validatingwebhook.yaml | 5 +- 5 files changed, 156 insertions(+), 7 deletions(-) create mode 100755 deploy/create-cert-key-secret.sh create mode 100644 deploy/kubefledged-service-webhook-server.yaml diff --git a/Makefile b/Makefile index b31f27a3..8039d77d 100644 --- a/Makefile +++ b/Makefile @@ -172,12 +172,15 @@ test: deploy-using-yaml: -kubectl apply -f deploy/kubefledged-namespace.yaml + bash deploy/create-cert-key-secret.sh --namespace kube-fledged --service kubefledged-webhook-server --secret kubefledged-webhook-server && \ kubectl apply -f deploy/kubefledged-crd.yaml && \ kubectl apply -f deploy/kubefledged-serviceaccount.yaml && \ kubectl apply -f deploy/kubefledged-clusterrole.yaml && \ kubectl apply -f deploy/kubefledged-clusterrolebinding.yaml && \ kubectl apply -f deploy/kubefledged-deployment-fledged-controller.yaml && \ - kubectl apply -f deploy/kubefledged-deployment-webhook-server.yaml + kubectl apply -f deploy/kubefledged-deployment-webhook-server.yaml && \ + kubectl apply -f deploy/kubefledged-service-webhook-server.yaml && \ + kubectl apply -f deploy/kubefledged-validatingwebhook.yaml deploy-using-operator: # Deploy the operator to a separate namespace called "operators" @@ -205,7 +208,8 @@ remove: kubectl delete -f deploy/kubefledged-namespace.yaml && \ kubectl delete -f deploy/kubefledged-clusterrolebinding.yaml && \ kubectl delete -f deploy/kubefledged-clusterrole.yaml && \ - kubectl delete -f deploy/kubefledged-crd.yaml + kubectl delete -f deploy/kubefledged-crd.yaml && \ + kubectl delete -f deploy/kubefledged-validatingwebhook.yaml remove-all: # Remove kube-fledged and the namespace "kube-fledged" diff --git a/deploy/create-cert-key-secret.sh b/deploy/create-cert-key-secret.sh new file mode 100755 index 00000000..5714434e --- /dev/null +++ b/deploy/create-cert-key-secret.sh @@ -0,0 +1,131 @@ +#!/bin/bash + +set -e + +usage() { + cat <> ${tmpdir}/csr.conf +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +[req_distinguished_name] +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = @alt_names +[alt_names] +DNS.1 = ${service} +DNS.2 = ${service}.${namespace} +DNS.3 = ${service}.${namespace}.svc +EOF + +openssl genrsa -out ${tmpdir}/server-key.pem 2048 +openssl req -new -key ${tmpdir}/server-key.pem -subj "/CN=${service}.${namespace}.svc" -out ${tmpdir}/server.csr -config ${tmpdir}/csr.conf + +# clean-up any previously created CSR for our service. Ignore errors if not present. +kubectl delete csr ${csrName} 2>/dev/null || true + +# create server cert/key CSR and send to k8s API +cat <&2 + exit 1 +fi +echo ${serverCert} | openssl base64 -d -A -out ${tmpdir}/server-cert.pem + + +# create the secret with CA cert and server cert/key +kubectl create secret generic ${secret} \ + --from-file=key.pem=${tmpdir}/server-key.pem \ + --from-file=cert.pem=${tmpdir}/server-cert.pem \ + --dry-run -o yaml | + kubectl -n ${namespace} apply -f - + diff --git a/deploy/kubefledged-deployment-webhook-server.yaml b/deploy/kubefledged-deployment-webhook-server.yaml index a3966807..933923c1 100644 --- a/deploy/kubefledged-deployment-webhook-server.yaml +++ b/deploy/kubefledged-deployment-webhook-server.yaml @@ -7,17 +7,16 @@ spec: replicas: 1 selector: matchLabels: - app: kubefledged + app: kubefledged-webhook-server template: metadata: labels: - app: kubefledged + app: kubefledged-webhook-server spec: containers: - image: senthilrch/fledged-webhook-server:v0.7.0 command: ["/opt/bin/fledged-webhook-server"] args: - - "--stderrthreshold=INFO" - "--cert-file=/var/run/secrets/webhook-server/cert.pem" - "--key-file=/var/run/secrets/webhook-server/key.pem" - "--port=443" diff --git a/deploy/kubefledged-service-webhook-server.yaml b/deploy/kubefledged-service-webhook-server.yaml new file mode 100644 index 00000000..c68a4ac7 --- /dev/null +++ b/deploy/kubefledged-service-webhook-server.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: kubefledged-webhook-server + namespace: kube-fledged +spec: + ports: + - name: webhook-server + port: 3443 + protocol: TCP + targetPort: 443 + selector: + app: kubefledged-webhook-server + type: ClusterIP diff --git a/deploy/kubefledged-validatingwebhook.yaml b/deploy/kubefledged-validatingwebhook.yaml index 4c018193..45c6eca3 100644 --- a/deploy/kubefledged-validatingwebhook.yaml +++ b/deploy/kubefledged-validatingwebhook.yaml @@ -3,16 +3,17 @@ kind: ValidatingWebhookConfiguration metadata: name: kubefledged webhooks: - - name: validate-image-cache + - name: validate-image-cache.fledged.k8s.io admissionReviewVersions: ["v1"] timeoutSeconds: 1 failurePolicy: Fail + sideEffects: None clientConfig: service: namespace: kube-fledged name: kubefledged-webhook-server path: "/validate-image-cache" - port: 443 + port: 3443 rules: - operations: ["CREATE", "UPDATE"] apiGroups: ["fledged.k8s.io"] From 0c739a835589c8357526f43fbead9dd3b4c62c53 Mon Sep 17 00:00:00 2001 From: bhuvanessr Date: Thu, 28 May 2020 14:11:40 +0530 Subject: [PATCH 12/21] script to patch ca bundle --- Makefile | 3 ++- deploy/kubefledged-validatingwebhook.yaml | 1 + ...-cert-key-secret.sh => webhook-create-signed-cert.sh} | 0 deploy/webhook-patch-ca-bundle.sh | 9 +++++++++ 4 files changed, 12 insertions(+), 1 deletion(-) rename deploy/{create-cert-key-secret.sh => webhook-create-signed-cert.sh} (100%) mode change 100755 => 100644 create mode 100644 deploy/webhook-patch-ca-bundle.sh diff --git a/Makefile b/Makefile index 8039d77d..717aa3f8 100644 --- a/Makefile +++ b/Makefile @@ -172,7 +172,8 @@ test: deploy-using-yaml: -kubectl apply -f deploy/kubefledged-namespace.yaml - bash deploy/create-cert-key-secret.sh --namespace kube-fledged --service kubefledged-webhook-server --secret kubefledged-webhook-server && \ + bash deploy/webhook-create-signed-cert.sh --namespace kube-fledged --service kubefledged-webhook-server --secret kubefledged-webhook-server && \ + bash deploy/webhook-patch-ca-bundle.sh && \ kubectl apply -f deploy/kubefledged-crd.yaml && \ kubectl apply -f deploy/kubefledged-serviceaccount.yaml && \ kubectl apply -f deploy/kubefledged-clusterrole.yaml && \ diff --git a/deploy/kubefledged-validatingwebhook.yaml b/deploy/kubefledged-validatingwebhook.yaml index 45c6eca3..5e31ef5c 100644 --- a/deploy/kubefledged-validatingwebhook.yaml +++ b/deploy/kubefledged-validatingwebhook.yaml @@ -14,6 +14,7 @@ webhooks: name: kubefledged-webhook-server path: "/validate-image-cache" port: 3443 + caBundle: ${CA_BUNDLE} rules: - operations: ["CREATE", "UPDATE"] apiGroups: ["fledged.k8s.io"] diff --git a/deploy/create-cert-key-secret.sh b/deploy/webhook-create-signed-cert.sh old mode 100755 new mode 100644 similarity index 100% rename from deploy/create-cert-key-secret.sh rename to deploy/webhook-create-signed-cert.sh diff --git a/deploy/webhook-patch-ca-bundle.sh b/deploy/webhook-patch-ca-bundle.sh new file mode 100644 index 00000000..dbb02719 --- /dev/null +++ b/deploy/webhook-patch-ca-bundle.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +export CA_BUNDLE=$(kubectl config view --raw --flatten -o json | jq -r '.clusters[] | select(.name == "'$(kubectl config current-context)'") | .cluster."certificate-authority-data"') + +sed -i "s|\${CA_BUNDLE}|${CA_BUNDLE}|g" deploy/kubefledged-validatingwebhook.yaml From 099208908b7bc478ce6370920e5f45fda0f707db Mon Sep 17 00:00:00 2001 From: bhuvanessr Date: Thu, 28 May 2020 14:30:46 +0530 Subject: [PATCH 13/21] added info logs --- pkg/webhook/imagecache.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/webhook/imagecache.go b/pkg/webhook/imagecache.go index 6e49195e..8eb1efc5 100644 --- a/pkg/webhook/imagecache.go +++ b/pkg/webhook/imagecache.go @@ -35,7 +35,7 @@ const ( // MutateImageCache modifies image cache resource func MutateImageCache(ar v1.AdmissionReview) *v1.AdmissionResponse { - klog.V(2).Info("mutating custom resource") + klog.Info("mutating custom resource") cr := struct { metav1.ObjectMeta Data map[string]string @@ -66,7 +66,7 @@ func MutateImageCache(ar v1.AdmissionReview) *v1.AdmissionResponse { // ValidateImageCache validates image cache resource func ValidateImageCache(ar v1.AdmissionReview) *v1.AdmissionResponse { - klog.V(2).Info("admitting custom resource") + klog.Info("admitting custom resource") cr := struct { metav1.ObjectMeta Data map[string]string From 976e307f1d6ffc1f38fa6cbcc25667b5bad3c7ff Mon Sep 17 00:00:00 2001 From: bhuvanessr Date: Thu, 28 May 2020 18:22:02 +0530 Subject: [PATCH 14/21] validate image cache in webhook server --- Makefile | 4 +- cmd/fledged/app/controller.go | 44 ---- cmd/fledged/app/controller_helpers.go | 73 ------- cmd/fledged/app/controller_helpers_test.go | 192 ------------------ cmd/webhook-server/main.go | 23 ++- ...kubefledged-deployment-webhook-server.yaml | 1 + pkg/webhook/imagecache.go | 103 +++++++--- 7 files changed, 91 insertions(+), 349 deletions(-) delete mode 100644 cmd/fledged/app/controller_helpers.go delete mode 100644 cmd/fledged/app/controller_helpers_test.go diff --git a/Makefile b/Makefile index 717aa3f8..5d55a054 100644 --- a/Makefile +++ b/Makefile @@ -201,8 +201,10 @@ deploy-using-operator: kubectl create -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml update: - kubectl scale deployment kubefledged --replicas=0 -n kube-fledged && sleep 1 && \ + kubectl scale deployment kubefledged --replicas=0 -n kube-fledged && \ + kubectl scale deployment kubefledged-webhook-server --replicas=0 -n kube-fledged && sleep 1 && \ kubectl scale deployment kubefledged --replicas=1 -n kube-fledged && sleep 1 && \ + kubectl scale deployment kubefledged-webhook-server --replicas=1 -n kube-fledged && sleep 1 && \ kubectl get pods -l app=kubefledged -n kube-fledged remove: diff --git a/cmd/fledged/app/controller.go b/cmd/fledged/app/controller.go index 0299eeda..d9f891d0 100644 --- a/cmd/fledged/app/controller.go +++ b/cmd/fledged/app/controller.go @@ -452,20 +452,6 @@ func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error { return err } - err = validateCacheSpec(c, imageCache) - if err != nil { - status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed - status.Reason = fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed - status.Message = err.Error() - - if err := c.updateImageCacheStatus(imageCache, status); err != nil { - glog.Errorf("Error updating imagecache status to %s: %v", status.Status, err) - return err - } - - return err - } - if wqKey.WorkType == images.ImageCacheUpdate && wqKey.OldImageCache == nil { status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed status.Reason = fledgedv1alpha1.ImageCacheReasonOldImageCacheNotFound @@ -479,36 +465,6 @@ func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error { return fmt.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonOldImageCacheNotFound, fledgedv1alpha1.ImageCacheMessageOldImageCacheNotFound) } - if wqKey.WorkType == images.ImageCacheUpdate { - if len(wqKey.OldImageCache.Spec.CacheSpec) != len(imageCache.Spec.CacheSpec) { - status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed - status.Reason = fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed - status.Message = fledgedv1alpha1.ImageCacheMessageNotSupportedUpdates - - if err = c.updateImageCacheSpecAndStatus(imageCache, wqKey.OldImageCache.Spec, status); err != nil { - glog.Errorf("Error updating imagecache spec and status to %s: %v", status.Status, err) - return err - } - glog.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, "Mismatch in no. of image lists") - return fmt.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, "Mismatch in no. of image lists") - } - - for i := range wqKey.OldImageCache.Spec.CacheSpec { - if !reflect.DeepEqual(wqKey.OldImageCache.Spec.CacheSpec[i].NodeSelector, imageCache.Spec.CacheSpec[i].NodeSelector) { - status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed - status.Reason = fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed - status.Message = fledgedv1alpha1.ImageCacheMessageNotSupportedUpdates - - if err = c.updateImageCacheSpecAndStatus(imageCache, wqKey.OldImageCache.Spec, status); err != nil { - glog.Errorf("Error updating imagecache spec and status to %s: %v", status.Status, err) - return err - } - glog.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, "Mismatch in node selector") - return fmt.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, "Mismatch in node selector") - } - } - } - cacheSpec := imageCache.Spec.CacheSpec glog.V(4).Infof("cacheSpec: %+v", cacheSpec) var nodes []*corev1.Node diff --git a/cmd/fledged/app/controller_helpers.go b/cmd/fledged/app/controller_helpers.go deleted file mode 100644 index dbe08fbf..00000000 --- a/cmd/fledged/app/controller_helpers.go +++ /dev/null @@ -1,73 +0,0 @@ -/* -Copyright 2018 The kube-fledged authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "fmt" - - "github.com/golang/glog" - fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" -) - -func validateCacheSpec(c *Controller, imageCache *fledgedv1alpha1.ImageCache) error { - if imageCache == nil { - glog.Errorf("Unable to obtain reference to image cache") - return fmt.Errorf("Unable to obtain reference to image cache") - } - - cacheSpec := imageCache.Spec.CacheSpec - glog.V(4).Infof("cacheSpec: %+v", cacheSpec) - var nodes []*corev1.Node - var err error - - for _, i := range cacheSpec { - if len(i.Images) == 0 { - glog.Error("No images specified within image list") - return fmt.Errorf("No images specified within image list") - - } - - for m := range i.Images { - for p := 0; p < m; p++ { - if i.Images[p] == i.Images[m] { - glog.Errorf("Duplicate image names within image list: %s", i.Images[m]) - return fmt.Errorf("Duplicate image names within image list: %s", i.Images[m]) - } - } - } - - if len(i.NodeSelector) > 0 { - if nodes, err = c.nodesLister.List(labels.Set(i.NodeSelector).AsSelector()); err != nil { - glog.Errorf("Error listing nodes using nodeselector %+v: %v", i.NodeSelector, err) - return err - } - } else { - if nodes, err = c.nodesLister.List(labels.Everything()); err != nil { - glog.Errorf("Error listing nodes using nodeselector labels.Everything(): %v", err) - return err - } - } - glog.V(4).Infof("No. of nodes in %+v is %d", i.NodeSelector, len(nodes)) - if len(nodes) == 0 { - glog.Errorf("NodeSelector %s did not match any nodes.", labels.Set(i.NodeSelector).String()) - return fmt.Errorf("NodeSelector %s did not match any nodes", labels.Set(i.NodeSelector).String()) - } - } - return nil -} diff --git a/cmd/fledged/app/controller_helpers_test.go b/cmd/fledged/app/controller_helpers_test.go deleted file mode 100644 index 1c42c603..00000000 --- a/cmd/fledged/app/controller_helpers_test.go +++ /dev/null @@ -1,192 +0,0 @@ -/* -Copyright 2018 The kube-fledged authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "fmt" - "strings" - "testing" - - fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" - fledgedclientsetfake "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/fake" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - fakeclientset "k8s.io/client-go/kubernetes/fake" -) - -var ( - alwaysReady = func() bool { return true } -) - -func TestValidateCacheSpec(t *testing.T) { - //var fakekubeclientset *fakeclientset.Clientset - //var fakefledgedclientset *fledgedclientsetfake.Clientset - - tests := []struct { - name string - imageCache *fledgedv1alpha1.ImageCache - nodeList *corev1.NodeList - nodeListError error - expectErr bool - errorString string - }{ - { - name: "Unable to obtain reference to image cache", - imageCache: nil, - nodeList: nil, - nodeListError: nil, - expectErr: true, - errorString: "Unable to obtain reference to image cache", - }, - { - name: "No images specified within image list", - imageCache: &fledgedv1alpha1.ImageCache{ - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{}, - }, - }, - }, - }, - nodeList: nil, - nodeListError: nil, - expectErr: true, - errorString: "No images specified within image list", - }, - { - name: "Duplicate image names within image list", - imageCache: &fledgedv1alpha1.ImageCache{ - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo", "foo"}, - }, - }, - }, - }, - nodeList: nil, - nodeListError: nil, - expectErr: true, - errorString: "Duplicate image names within image list", - }, - { - name: "Error listing nodes using nodeselector", - imageCache: &fledgedv1alpha1.ImageCache{ - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - NodeSelector: map[string]string{"foo": "bar"}, - }, - }, - }, - }, - nodeList: nil, - nodeListError: fmt.Errorf("fake error"), - expectErr: true, - errorString: "Error listing nodes using nodeselector", - }, - { - name: "Error listing nodes using nodeselector labels.Everything()", - imageCache: &fledgedv1alpha1.ImageCache{ - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - }, - }, - }, - }, - nodeList: nil, - nodeListError: fmt.Errorf("fake error"), - expectErr: true, - errorString: "Error listing nodes using nodeselector labels.Everything()", - }, - { - name: "NodeSelector did not match any nodes", - imageCache: &fledgedv1alpha1.ImageCache{ - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - NodeSelector: map[string]string{"foo": "bar"}, - }, - }, - }, - }, - nodeList: &corev1.NodeList{}, - nodeListError: nil, - expectErr: true, - errorString: "NodeSelector foo=bar did not match any nodes", - }, - { - name: "Successful validation", - imageCache: &fledgedv1alpha1.ImageCache{ - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo"}, - NodeSelector: map[string]string{"foo": "bar"}, - }, - }, - }, - }, - nodeList: &corev1.NodeList{ - Items: []corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "fakenode", - Labels: map[string]string{"foo": "bar"}, - }, - }, - }, - }, - nodeListError: nil, - expectErr: false, - errorString: "", - }, - } - - for _, test := range tests { - fakekubeclientset := &fakeclientset.Clientset{} - fakefledgedclientset := &fledgedclientsetfake.Clientset{} - - controller, nodeInformer, _ := newTestController(fakekubeclientset, fakefledgedclientset) - - if test.nodeListError != nil { - //TODO: How to return a fake error from node Lister? - continue - } - - if test.nodeList != nil && len(test.nodeList.Items) > 0 { - for _, node := range test.nodeList.Items { - nodeInformer.Informer().GetIndexer().Add(&node) - } - } - - err := validateCacheSpec(controller, test.imageCache) - if test.expectErr { - if err != nil && strings.HasPrefix(err.Error(), test.errorString) { - } else { - t.Errorf("Test: %s failed", test.name) - } - } else if err != nil { - t.Errorf("Test: %s failed. err received = %s", test.name, err.Error()) - } - } -} diff --git a/cmd/webhook-server/main.go b/cmd/webhook-server/main.go index aa3040ea..9fce1c43 100644 --- a/cmd/webhook-server/main.go +++ b/cmd/webhook-server/main.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "net/http" + "github.com/golang/glog" "github.com/senthilrch/kube-fledged/pkg/webhook" admissionv1 "k8s.io/api/admission/v1" @@ -36,7 +37,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/klog" // TODO: try this library to see if it generates correct json patch // https://github.com/mattbaird/jsonpatch ) @@ -89,7 +89,7 @@ type Config struct { func configTLS(config Config) *tls.Config { sCert, err := tls.LoadX509KeyPair(config.CertFile, config.KeyFile) if err != nil { - klog.Fatal(err) + glog.Fatal(err) } return &tls.Config{ Certificates: []tls.Certificate{sCert}, @@ -126,17 +126,17 @@ func serve(w http.ResponseWriter, r *http.Request, admit admitHandler) { // verify the content type is accurate contentType := r.Header.Get("Content-Type") if contentType != "application/json" { - klog.Errorf("contentType=%s, expect application/json", contentType) + glog.Errorf("contentType=%s, expect application/json", contentType) return } - klog.V(2).Info(fmt.Sprintf("handling request: %s", body)) + glog.V(2).Info(fmt.Sprintf("handling request: %s", body)) deserializer := codecs.UniversalDeserializer() obj, gvk, err := deserializer.Decode(body, nil, nil) if err != nil { msg := fmt.Sprintf("Request could not be decoded: %v", err) - klog.Error(msg) + glog.Error(msg) http.Error(w, msg, http.StatusBadRequest) return } @@ -146,7 +146,7 @@ func serve(w http.ResponseWriter, r *http.Request, admit admitHandler) { case v1beta1.SchemeGroupVersion.WithKind("AdmissionReview"): requestedAdmissionReview, ok := obj.(*v1beta1.AdmissionReview) if !ok { - klog.Errorf("Expected v1beta1.AdmissionReview but got: %T", obj) + glog.Errorf("Expected v1beta1.AdmissionReview but got: %T", obj) return } responseAdmissionReview := &v1beta1.AdmissionReview{} @@ -157,7 +157,7 @@ func serve(w http.ResponseWriter, r *http.Request, admit admitHandler) { case v1.SchemeGroupVersion.WithKind("AdmissionReview"): requestedAdmissionReview, ok := obj.(*v1.AdmissionReview) if !ok { - klog.Errorf("Expected v1.AdmissionReview but got: %T", obj) + glog.Errorf("Expected v1.AdmissionReview but got: %T", obj) return } responseAdmissionReview := &v1.AdmissionReview{} @@ -167,21 +167,21 @@ func serve(w http.ResponseWriter, r *http.Request, admit admitHandler) { responseObj = responseAdmissionReview default: msg := fmt.Sprintf("Unsupported group version kind: %v", gvk) - klog.Error(msg) + glog.Error(msg) http.Error(w, msg, http.StatusBadRequest) return } - klog.V(2).Info(fmt.Sprintf("sending response: %v", responseObj)) + glog.V(2).Info(fmt.Sprintf("sending response: %v", responseObj)) respBytes, err := json.Marshal(responseObj) if err != nil { - klog.Error(err) + glog.Error(err) http.Error(w, err.Error(), http.StatusInternalServerError) return } w.Header().Set("Content-Type", "application/json") if _, err := w.Write(respBytes); err != nil { - klog.Error(err) + glog.Error(err) } } @@ -279,6 +279,7 @@ func main() { Addr: fmt.Sprintf(":%d", port), TLSConfig: configTLS(config), } + glog.Infof("Wehook server listening on :%d", port) err := server.ListenAndServeTLS("", "") if err != nil { panic(err) diff --git a/deploy/kubefledged-deployment-webhook-server.yaml b/deploy/kubefledged-deployment-webhook-server.yaml index 933923c1..0527cd01 100644 --- a/deploy/kubefledged-deployment-webhook-server.yaml +++ b/deploy/kubefledged-deployment-webhook-server.yaml @@ -17,6 +17,7 @@ spec: - image: senthilrch/fledged-webhook-server:v0.7.0 command: ["/opt/bin/fledged-webhook-server"] args: + - "--stderrthreshold=INFO" - "--cert-file=/var/run/secrets/webhook-server/cert.pem" - "--key-file=/var/run/secrets/webhook-server/key.pem" - "--port=443" diff --git a/pkg/webhook/imagecache.go b/pkg/webhook/imagecache.go index 8eb1efc5..9f558dbf 100644 --- a/pkg/webhook/imagecache.go +++ b/pkg/webhook/imagecache.go @@ -18,10 +18,13 @@ package webhook import ( "encoding/json" + "fmt" + "reflect" + "github.com/golang/glog" + fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" v1 "k8s.io/api/admission/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/klog" ) const ( @@ -35,7 +38,7 @@ const ( // MutateImageCache modifies image cache resource func MutateImageCache(ar v1.AdmissionReview) *v1.AdmissionResponse { - klog.Info("mutating custom resource") + glog.V(4).Info("mutating custom resource") cr := struct { metav1.ObjectMeta Data map[string]string @@ -44,7 +47,7 @@ func MutateImageCache(ar v1.AdmissionReview) *v1.AdmissionResponse { raw := ar.Request.Object.Raw err := json.Unmarshal(raw, &cr) if err != nil { - klog.Error(err) + glog.Error(err) return toV1AdmissionResponse(err) } @@ -66,41 +69,85 @@ func MutateImageCache(ar v1.AdmissionReview) *v1.AdmissionResponse { // ValidateImageCache validates image cache resource func ValidateImageCache(ar v1.AdmissionReview) *v1.AdmissionResponse { - klog.Info("admitting custom resource") - cr := struct { - metav1.ObjectMeta - Data map[string]string - }{} + glog.V(4).Info("admitting image cache") + var raw, oldraw []byte + var imageCache, oldImageCache fledgedv1alpha1.ImageCache - var raw []byte - if ar.Request.Operation == v1.Delete { - raw = ar.Request.OldObject.Raw - } else { - raw = ar.Request.Object.Raw - } - err := json.Unmarshal(raw, &cr) + reviewResponse := v1.AdmissionResponse{} + reviewResponse.Allowed = true + + raw = ar.Request.Object.Raw + err := json.Unmarshal(raw, &imageCache) if err != nil { - klog.Error(err) + glog.Error(err) return toV1AdmissionResponse(err) } - reviewResponse := v1.AdmissionResponse{} - reviewResponse.Allowed = true - for k, v := range cr.Data { - if k == "webhook-e2e-test" && v == "webhook-disallow" && - (ar.Request.Operation == v1.Create || ar.Request.Operation == v1.Update) { - reviewResponse.Allowed = false - reviewResponse.Result = &metav1.Status{ - Reason: "the custom resource contains unwanted data", + if ar.Request.Operation == v1.Update { + oldraw = ar.Request.OldObject.Raw + err := json.Unmarshal(oldraw, &oldImageCache) + if err != nil { + glog.Error(err) + return toV1AdmissionResponse(err) + } + if reflect.DeepEqual(oldImageCache.Spec, imageCache.Spec) { + glog.V(4).Info("No change in image cache spec: skipping validation") + return &reviewResponse + } + } + + cacheSpec := imageCache.Spec.CacheSpec + glog.V(4).Infof("cacheSpec: %+v", cacheSpec) + + for _, i := range cacheSpec { + if len(i.Images) == 0 { + glog.Error("No images specified within image list") + return toV1AdmissionResponse(fmt.Errorf("No images specified within image list")) + } + + for m := range i.Images { + for p := 0; p < m; p++ { + if i.Images[p] == i.Images[m] { + glog.Errorf("Duplicate image names within image list: %s", i.Images[m]) + return toV1AdmissionResponse(fmt.Errorf("Duplicate image names within image list: %s", i.Images[m])) + } + } + } + /* + if len(i.NodeSelector) > 0 { + if nodes, err = c.nodesLister.List(labels.Set(i.NodeSelector).AsSelector()); err != nil { + glog.Errorf("Error listing nodes using nodeselector %+v: %v", i.NodeSelector, err) + return err + } + } else { + if nodes, err = c.nodesLister.List(labels.Everything()); err != nil { + glog.Errorf("Error listing nodes using nodeselector labels.Everything(): %v", err) + return err + } + } + glog.V(4).Infof("No. of nodes in %+v is %d", i.NodeSelector, len(nodes)) + if len(nodes) == 0 { + glog.Errorf("NodeSelector %s did not match any nodes.", labels.Set(i.NodeSelector).String()) + return fmt.Errorf("NodeSelector %s did not match any nodes", labels.Set(i.NodeSelector).String()) } + */ + } + + if ar.Request.Operation == v1.Update { + if len(oldImageCache.Spec.CacheSpec) != len(imageCache.Spec.CacheSpec) { + glog.Errorf("Mismatch in no. of image lists") + return toV1AdmissionResponse(fmt.Errorf("Mismatch in no. of image lists")) } - if k == "webhook-e2e-test" && v == "webhook-nondeletable" && ar.Request.Operation == v1.Delete { - reviewResponse.Allowed = false - reviewResponse.Result = &metav1.Status{ - Reason: "the custom resource cannot be deleted because it contains unwanted key and value", + + for i := range oldImageCache.Spec.CacheSpec { + if !reflect.DeepEqual(oldImageCache.Spec.CacheSpec[i].NodeSelector, imageCache.Spec.CacheSpec[i].NodeSelector) { + glog.Errorf("Mismatch in node selector") + return toV1AdmissionResponse(fmt.Errorf("Mismatch in node selector")) } } } + + glog.Info("Image cache creation/update validated successfully") return &reviewResponse } From 18d997c8099f2d3bf29a4903761fbab63604382b Mon Sep 17 00:00:00 2001 From: bhuvanessr Date: Thu, 28 May 2020 22:34:31 +0530 Subject: [PATCH 15/21] make changes to build and ci --- .travis.yml | 7 +- Makefile | 92 +++++++++---------- README.md | 4 +- ...ckerfile.fledged => Dockerfile.controller} | 15 ++- ....fledged_dev => Dockerfile.controller_dev} | 6 +- ...le.docker_client => Dockerfile.cri_client} | 0 build/Dockerfile.webhook_server | 15 ++- build/Dockerfile.webhook_server_dev | 6 +- cmd/{fledged => controller}/app/controller.go | 0 .../app/controller_test.go | 8 +- cmd/{fledged => controller}/main.go | 4 +- cmd/webhook-server/main.go | 2 +- deploy/kubefledged-clusterrolebinding.yaml | 2 +- deploy/kubefledged-crd.yaml | 4 +- ...=> kubefledged-deployment-controller.yaml} | 18 ++-- ...kubefledged-deployment-webhook-server.yaml | 4 +- deploy/kubefledged-imagecache.yaml | 2 +- deploy/kubefledged-serviceaccount.yaml | 2 +- deploy/kubefledged-validatingwebhook.yaml | 4 +- deploy/webhook-create-signed-cert.sh | 14 +++ deploy/webhook-patch-ca-bundle.sh | 14 +++ pkg/webhook/imagecache.go | 2 + 22 files changed, 122 insertions(+), 103 deletions(-) rename build/{Dockerfile.fledged => Dockerfile.controller} (62%) rename build/{Dockerfile.fledged_dev => Dockerfile.controller_dev} (81%) rename build/{Dockerfile.docker_client => Dockerfile.cri_client} (100%) rename cmd/{fledged => controller}/app/controller.go (100%) rename cmd/{fledged => controller}/app/controller_test.go (99%) rename cmd/{fledged => controller}/main.go (93%) rename deploy/{kubefledged-deployment-fledged-controller.yaml => kubefledged-deployment-controller.yaml} (58%) diff --git a/.travis.yml b/.travis.yml index b32f5476..cf19dad2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,16 +39,17 @@ jobs: - hack/verify-gofmt.sh - hack/verify-golint.sh - hack/verify-govet.sh - - make fledged-amd64 + # BUILD_OUTPUT is left empty to indicate buildx to leave the built images in it's cache + - travis_wait 30 make BUILD_OUTPUT= controller-amd64 webhook-server-amd64 - make test - $(go env GOPATH | awk 'BEGIN{FS=":"} {print $1}')/bin/goveralls -coverprofile=coverage.out -service=travis-ci - stage: build_multiarch script: # BUILD_OUTPUT is left empty to indicate buildx to leave the built images in it's cache - - travis_wait 60 make GIT_BRANCH=${TRAVIS_BRANCH} BUILD_OUTPUT= release + - travis_wait 60 make BUILD_OUTPUT= release - stage: build_release script: - docker login -u ${DOCKERHUB_USER} -p ${DOCKERHUB_PSWD} # BUILD_OUTPUT=--push requests buildx to push the built images to docker hub - - travis_wait 60 make GIT_BRANCH=${TRAVIS_BRANCH} BUILD_OUTPUT=--push release + - travis_wait 60 make BUILD_OUTPUT=--push release diff --git a/Makefile b/Makefile index 5d55a054..82ba7916 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -.PHONY: clean clean-fledged clean-client clean-operator fledged-amd64 fledged-image client-image operator-image build-images push-images test deploy update remove +.PHONY: clean clean-controller clean-cri-client clean-operator controller-amd64 controller-image cri-client-image operator-image build-images push-images test deploy update remove # Default tag and architecture. Can be overridden TAG?=$(shell git describe --tags --dirty) ARCH?=amd64 @@ -26,16 +26,16 @@ endif GOARM=7 DOCKER_CLI_EXPERIMENTAL=enabled -ifndef FLEDGED_IMAGE_REPO - FLEDGED_IMAGE_REPO=docker.io/senthilrch/fledged +ifndef CONTROLLER_IMAGE_REPO + CONTROLLER_IMAGE_REPO=docker.io/senthilrch/kubefledged-controller endif -ifndef FLEDGED_WEBHOOK_SERVER_IMAGE_REPO - FLEDGED_WEBHOOK_SERVER_IMAGE_REPO=docker.io/senthilrch/fledged-webhook-server +ifndef WEBHOOK_SERVER_IMAGE_REPO + WEBHOOK_SERVER_IMAGE_REPO=docker.io/senthilrch/kubefledged-webhook-server endif -ifndef FLEDGED_DOCKER_CLIENT_IMAGE_REPO - FLEDGED_DOCKER_CLIENT_IMAGE_REPO=docker.io/senthilrch/fledged-docker-client +ifndef CRI_CLIENT_IMAGE_REPO + CRI_CLIENT_IMAGE_REPO=docker.io/senthilrch/kubefledged-cri-client endif ifndef OPERATOR_IMAGE_REPO @@ -66,10 +66,6 @@ ifndef OPERATORSDK_VERSION OPERATORSDK_VERSION=v0.17.0 endif -ifndef GIT_BRANCH - GIT_BRANCH=master -endif - ifndef TARGET_PLATFORMS TARGET_PLATFORMS=linux/amd64,linux/arm/v7,linux/arm64/v8 endif @@ -94,70 +90,66 @@ endif ### BUILD -clean: clean-fledged clean-webhook-server clean-client clean-operator +clean: clean-controller clean-webhook-server clean-cri-client clean-operator -clean-fledged: - -rm -f build/fledged - -docker image rm ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION} +clean-controller: + -rm -f build/kubefledged-controller + -docker image rm ${CONTROLLER_IMAGE_REPO}:${RELEASE_VERSION} -docker image rm `docker image ls -f dangling=true -q` clean-webhook-server: - -rm -f build/webhook-server - -docker image rm ${FLEDGED_WEBHOOK_SERVER_IMAGE_REPO}:${RELEASE_VERSION} + -rm -f build/kubefledged-webhook-server + -docker image rm ${WEBHOOK_SERVER_IMAGE_REPO}:${RELEASE_VERSION} -docker image rm `docker image ls -f dangling=true -q` -clean-client: - -docker image rm ${FLEDGED_DOCKER_CLIENT_IMAGE_REPO}:${RELEASE_VERSION} +clean-cri-client: + -docker image rm ${CRI_CLIENT_IMAGE_REPO}:${RELEASE_VERSION} -docker image rm `docker image ls -f dangling=true -q` clean-operator: -docker image rm ${OPERATOR_IMAGE_REPO}:${RELEASE_VERSION} -docker image rm `docker image ls -f dangling=true -q` -fledged-amd64: clean-fledged - CGO_ENABLED=0 go build -o build/fledged -ldflags '-s -w -extldflags "-static"' cmd/fledged/main.go && \ - cd build && docker build -t ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION} -f Dockerfile.fledged_dev \ - --build-arg ALPINE_VERSION=${ALPINE_VERSION} . +controller-image: clean-controller + docker buildx build --platform=${TARGET_PLATFORMS} -t ${CONTROLLER_IMAGE_REPO}:${RELEASE_VERSION} \ + -f build/Dockerfile.controller ${HTTP_PROXY_CONFIG} ${HTTPS_PROXY_CONFIG} \ + --build-arg GOLANG_VERSION=${GOLANG_VERSION} --build-arg ALPINE_VERSION=${ALPINE_VERSION} --progress=plain ${BUILD_OUTPUT} . -fledged-dev: fledged-amd64 - docker push ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION} +controller-amd64: TARGET_PLATFORMS=linux/amd64 +controller-amd64: install-buildx controller-image -webhook-server-amd64: clean-webhook-server - CGO_ENABLED=0 go build -o build/fledged-webhook-server -ldflags '-s -w -extldflags "-static"' cmd/webhook-server/main.go && \ - cd build && docker build -t ${FLEDGED_WEBHOOK_SERVER_IMAGE_REPO}:${RELEASE_VERSION} -f Dockerfile.webhook_server_dev \ +controller-dev: clean-controller + CGO_ENABLED=0 go build -o build/kubefledged-controller -ldflags '-s -w -extldflags "-static"' cmd/controller/main.go && \ + docker build -t ${CONTROLLER_IMAGE_REPO}:${RELEASE_VERSION} -f build/Dockerfile.controller_dev \ --build-arg ALPINE_VERSION=${ALPINE_VERSION} . - -webhook-server-dev: webhook-server-amd64 - docker push ${FLEDGED_WEBHOOK_SERVER_IMAGE_REPO}:${RELEASE_VERSION} - -fledged-image: clean-fledged - cd build && docker buildx build --platform=${TARGET_PLATFORMS} -t ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION} \ - -f Dockerfile.fledged ${HTTP_PROXY_CONFIG} ${HTTPS_PROXY_CONFIG} --build-arg GIT_BRANCH=${GIT_BRANCH} \ - --build-arg GOLANG_VERSION=${GOLANG_VERSION} --build-arg ALPINE_VERSION=${ALPINE_VERSION} --progress=plain ${BUILD_OUTPUT} . + docker push ${CONTROLLER_IMAGE_REPO}:${RELEASE_VERSION} webhook-server-image: clean-webhook-server - cd build && docker buildx build --platform=${TARGET_PLATFORMS} -t ${FLEDGED_WEBHOOK_SERVER_IMAGE_REPO}:${RELEASE_VERSION} \ - -f Dockerfile.webhook_server ${HTTP_PROXY_CONFIG} ${HTTPS_PROXY_CONFIG} --build-arg GIT_BRANCH=${GIT_BRANCH} \ + docker buildx build --platform=${TARGET_PLATFORMS} -t ${WEBHOOK_SERVER_IMAGE_REPO}:${RELEASE_VERSION} \ + -f build/Dockerfile.webhook_server ${HTTP_PROXY_CONFIG} ${HTTPS_PROXY_CONFIG} \ --build-arg GOLANG_VERSION=${GOLANG_VERSION} --build-arg ALPINE_VERSION=${ALPINE_VERSION} --progress=plain ${BUILD_OUTPUT} . -client-image: clean-client - cd build && docker buildx build --platform=${TARGET_PLATFORMS} -t ${FLEDGED_DOCKER_CLIENT_IMAGE_REPO}:${RELEASE_VERSION} \ - -f Dockerfile.docker_client ${HTTP_PROXY_CONFIG} ${HTTPS_PROXY_CONFIG} \ +webhook-server-amd64: TARGET_PLATFORMS=linux/amd64 +webhook-server-amd64: install-buildx webhook-server-image + +webhook-server-dev: clean-webhook-server + CGO_ENABLED=0 go build -o build/kubefledged-webhook-server -ldflags '-s -w -extldflags "-static"' cmd/webhook-server/main.go && \ + docker build -t ${WEBHOOK_SERVER_IMAGE_REPO}:${RELEASE_VERSION} -f build/Dockerfile.webhook_server_dev \ + --build-arg ALPINE_VERSION=${ALPINE_VERSION} . + docker push ${WEBHOOK_SERVER_IMAGE_REPO}:${RELEASE_VERSION} + +cri-client-image: clean-cri-client + docker buildx build --platform=${TARGET_PLATFORMS} -t ${CRI_CLIENT_IMAGE_REPO}:${RELEASE_VERSION} \ + -f build/Dockerfile.cri_client ${HTTP_PROXY_CONFIG} ${HTTPS_PROXY_CONFIG} \ --build-arg DOCKER_VERSION=${DOCKER_VERSION} --build-arg CRICTL_VERSION=${CRICTL_VERSION} \ --build-arg ALPINE_VERSION=${ALPINE_VERSION} --progress=plain ${BUILD_OUTPUT} . operator-image: clean-operator cd deploy/kubefledged-operator && \ docker buildx build --platform=${OPERATOR_TARGET_PLATFORMS} -t ${OPERATOR_IMAGE_REPO}:${RELEASE_VERSION} \ - -f ./build/Dockerfile --build-arg OPERATORSDK_VERSION=${OPERATORSDK_VERSION} --progress=plain ${BUILD_OUTPUT} . - -push-images: - -docker push ${FLEDGED_IMAGE_REPO}:${RELEASE_VERSION} - -docker push ${FLEDGED_WEBHOOK_SERVER_IMAGE_REPO}:${RELEASE_VERSION} - -docker push ${FLEDGED_DOCKER_CLIENT_IMAGE_REPO}:${RELEASE_VERSION} - -docker push ${OPERATOR_IMAGE_REPO}:${RELEASE_VERSION} + -f build/Dockerfile --build-arg OPERATORSDK_VERSION=${OPERATORSDK_VERSION} --progress=plain ${BUILD_OUTPUT} . -release: install-buildx fledged-image webhook-server-image client-image operator-image +release: install-buildx controller-image webhook-server-image cri-client-image operator-image install-buildx: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes @@ -178,7 +170,7 @@ deploy-using-yaml: kubectl apply -f deploy/kubefledged-serviceaccount.yaml && \ kubectl apply -f deploy/kubefledged-clusterrole.yaml && \ kubectl apply -f deploy/kubefledged-clusterrolebinding.yaml && \ - kubectl apply -f deploy/kubefledged-deployment-fledged-controller.yaml && \ + kubectl apply -f deploy/kubefledged-deployment-controller.yaml && \ kubectl apply -f deploy/kubefledged-deployment-webhook-server.yaml && \ kubectl apply -f deploy/kubefledged-service-webhook-server.yaml && \ kubectl apply -f deploy/kubefledged-validatingwebhook.yaml diff --git a/README.md b/README.md index bed3329d..f5921e6e 100644 --- a/README.md +++ b/README.md @@ -256,13 +256,13 @@ _fledged_ has a built-in image manager routine that is responsible for pulling a For more detailed description, go through _kube-fledged's_ [design proposal](docs/cluster-image-cache.md). -## Configuration Flags +## Configuration Flags for Kubefledged Controller `--image-pull-deadline-duration:` Maximum duration allowed for pulling an image. After this duration, image pull is considered to have failed. default "5m" `--image-cache-refresh-frequency:` The image cache is refreshed periodically to ensure the cache is up to date. Setting this flag to "0s" will disable refresh. default "15m" -`--docker-client-image:` The image name of the docker client. The docker client is used when deleting images during purging the cache". +`--cri-client-image:` The image name of the cri client. The cri client is used when deleting images during purging the cache". `--image-pull-policy:` Image pull policy for pulling images into and refreshing the cache. Possible values are 'IfNotPresent' and 'Always'. Default value is 'IfNotPresent'. Image with no or ":latest" tag are always pulled. diff --git a/build/Dockerfile.fledged b/build/Dockerfile.controller similarity index 62% rename from build/Dockerfile.fledged rename to build/Dockerfile.controller index 3748e3fd..cf6abd1a 100644 --- a/build/Dockerfile.fledged +++ b/build/Dockerfile.controller @@ -17,14 +17,13 @@ ARG ALPINE_VERSION FROM golang:$GOLANG_VERSION AS builder LABEL stage=builder -ARG GIT_BRANCH -RUN mkdir -p /go/src/github.com/senthilrch && \ - git clone --depth=1 --single-branch --branch=$GIT_BRANCH https://github.com/senthilrch/kube-fledged.git /go/src/github.com/senthilrch/kube-fledged && \ - cd /go/src/github.com/senthilrch/kube-fledged && \ - CGO_ENABLED=0 go build -o build/fledged -ldflags '-s -w -extldflags "-static"' cmd/fledged/main.go +RUN mkdir -p /go/src/github.com/senthilrch/kube-fledged +COPY . /go/src/github.com/senthilrch/kube-fledged +WORKDIR /go/src/github.com/senthilrch/kube-fledged +RUN CGO_ENABLED=0 go build -o build/kubefledged-controller -ldflags '-s -w -extldflags "-static"' cmd/controller/main.go FROM alpine:$ALPINE_VERSION LABEL maintainer="senthilrch " -COPY --from=builder /go/src/github.com/senthilrch/kube-fledged/build/fledged /opt/bin/fledged -RUN chmod 755 /opt/bin/fledged -ENTRYPOINT ["/opt/bin/fledged"] +COPY --from=builder /go/src/github.com/senthilrch/kube-fledged/build/kubefledged-controller /opt/bin/kubefledged-controller +RUN chmod 755 /opt/bin/kubefledged-controller +ENTRYPOINT ["/opt/bin/kubefledged-controller"] diff --git a/build/Dockerfile.fledged_dev b/build/Dockerfile.controller_dev similarity index 81% rename from build/Dockerfile.fledged_dev rename to build/Dockerfile.controller_dev index d002f012..dd7e85a3 100644 --- a/build/Dockerfile.fledged_dev +++ b/build/Dockerfile.controller_dev @@ -17,7 +17,7 @@ ARG ALPINE_VERSION FROM alpine:$ALPINE_VERSION LABEL maintainer="senthilrch " -COPY fledged /opt/bin/fledged -RUN chmod 755 /opt/bin/fledged +COPY build/kubefledged-controller /opt/bin/kubefledged-controller +RUN chmod 755 /opt/bin/kubefledged-controller -ENTRYPOINT ["/opt/bin/fledged"] +ENTRYPOINT ["/opt/bin/kubefledged-controller"] diff --git a/build/Dockerfile.docker_client b/build/Dockerfile.cri_client similarity index 100% rename from build/Dockerfile.docker_client rename to build/Dockerfile.cri_client diff --git a/build/Dockerfile.webhook_server b/build/Dockerfile.webhook_server index 9cf8ea49..07de00b9 100644 --- a/build/Dockerfile.webhook_server +++ b/build/Dockerfile.webhook_server @@ -17,14 +17,13 @@ ARG ALPINE_VERSION FROM golang:$GOLANG_VERSION AS builder LABEL stage=builder -ARG GIT_BRANCH -RUN mkdir -p /go/src/github.com/senthilrch && \ - git clone --depth=1 --single-branch --branch=$GIT_BRANCH https://github.com/senthilrch/kube-fledged.git /go/src/github.com/senthilrch/kube-fledged && \ - cd /go/src/github.com/senthilrch/kube-fledged && \ - CGO_ENABLED=0 go build -o build/fledged-webhook-server -ldflags '-s -w -extldflags "-static"' cmd/webhook-server/main.go +RUN mkdir -p /go/src/github.com/senthilrch/kube-fledged +COPY . /go/src/github.com/senthilrch/kube-fledged +WORKDIR /go/src/github.com/senthilrch/kube-fledged +RUN CGO_ENABLED=0 go build -o build/kubefledged-webhook-server -ldflags '-s -w -extldflags "-static"' cmd/webhook-server/main.go FROM alpine:$ALPINE_VERSION LABEL maintainer="senthilrch " -COPY --from=builder /go/src/github.com/senthilrch/kube-fledged/build/fledged-webhook-server /opt/bin/fledged-webhook-server -RUN chmod 755 /opt/bin/fledged-webhook-server -ENTRYPOINT ["/opt/bin/fledged-webhook-server"] +COPY --from=builder /go/src/github.com/senthilrch/kube-fledged/build/kubefledged-webhook-server /opt/bin/kubefledged-webhook-server +RUN chmod 755 /opt/bin/kubefledged-webhook-server +ENTRYPOINT ["/opt/bin/kubefledged-webhook-server"] diff --git a/build/Dockerfile.webhook_server_dev b/build/Dockerfile.webhook_server_dev index 08def573..9f3904bf 100644 --- a/build/Dockerfile.webhook_server_dev +++ b/build/Dockerfile.webhook_server_dev @@ -17,7 +17,7 @@ ARG ALPINE_VERSION FROM alpine:$ALPINE_VERSION LABEL maintainer="senthilrch " -COPY fledged-webhook-server /opt/bin/fledged-webhook-server -RUN chmod 755 /opt/bin/fledged-webhook-server +COPY build/kubefledged-webhook-server /opt/bin/kubefledged-webhook-server +RUN chmod 755 /opt/bin/kubefledged-webhook-server -ENTRYPOINT ["/opt/bin/fledged-webhook-server"] +ENTRYPOINT ["/opt/bin/kubefledged-webhook-server"] diff --git a/cmd/fledged/app/controller.go b/cmd/controller/app/controller.go similarity index 100% rename from cmd/fledged/app/controller.go rename to cmd/controller/app/controller.go diff --git a/cmd/fledged/app/controller_test.go b/cmd/controller/app/controller_test.go similarity index 99% rename from cmd/fledged/app/controller_test.go rename to cmd/controller/app/controller_test.go index 8befbcf2..b6b0a8c5 100644 --- a/cmd/fledged/app/controller_test.go +++ b/cmd/controller/app/controller_test.go @@ -472,7 +472,7 @@ func TestSyncHandler(t *testing.T) { expectErr: true, expectedErrString: "unexpected key format", }, - { + /*{ name: "#2: Create - Invalid imagecache spec (no images specified)", imageCache: fledgedv1alpha1.ImageCache{ ObjectMeta: metav1.ObjectMeta{ @@ -494,7 +494,7 @@ func TestSyncHandler(t *testing.T) { expectedActions: []ActionReaction{{action: "update", reaction: ""}}, expectErr: true, expectedErrString: "No images specified within image list", - }, + },*/ { name: "#3: Update - Old imagecache pointer is nil", imageCache: defaultImageCache, @@ -508,7 +508,7 @@ func TestSyncHandler(t *testing.T) { expectErr: true, expectedErrString: "OldImageCacheNotFound", }, - { + /*{ name: "#4: Update - No. of imagelists not equal", imageCache: defaultImageCache, wqKey: images.WorkQueueKey{ @@ -561,7 +561,7 @@ func TestSyncHandler(t *testing.T) { expectedActions: []ActionReaction{{action: "update", reaction: ""}}, expectErr: true, expectedErrString: "CacheSpecValidationFailed", - }, + },*/ { name: "#6: Refresh - Update status to processing", imageCache: defaultImageCache, diff --git a/cmd/fledged/main.go b/cmd/controller/main.go similarity index 93% rename from cmd/fledged/main.go rename to cmd/controller/main.go index a112ab54..cbc923ae 100644 --- a/cmd/fledged/main.go +++ b/cmd/controller/main.go @@ -28,7 +28,7 @@ import ( // Uncomment the following line to load the gcp plugin (only required to authenticate against GKE clusters). // _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" - "github.com/senthilrch/kube-fledged/cmd/fledged/app" + "github.com/senthilrch/kube-fledged/cmd/controller/app" clientset "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned" informers "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions" "github.com/senthilrch/kube-fledged/pkg/signals" @@ -89,7 +89,7 @@ func main() { func init() { flag.DurationVar(&imagePullDeadlineDuration, "image-pull-deadline-duration", time.Minute*5, "Maximum duration allowed for pulling an image. After this duration, image pull is considered to have failed") flag.DurationVar(&imageCacheRefreshFrequency, "image-cache-refresh-frequency", time.Minute*15, "The image cache is refreshed periodically to ensure the cache is up to date. Setting this flag to 0s will disable refresh") - flag.StringVar(&dockerClientImage, "docker-client-image", "senthilrch/fledged-docker-client:latest", "The image name of the docker client. the docker client is used when deleting images during purging the cache") + flag.StringVar(&dockerClientImage, "cri-client-image", "senthilrch/kubefledged-cri-client:latest", "The image name of the cri client. the cri client is used when deleting images during purging the cache") flag.StringVar(&imagePullPolicy, "image-pull-policy", "IfNotPresent", "Image pull policy for pulling images into the cache. Possible values are 'IfNotPresent' and 'Always'. Default value is 'IfNotPresent'. Images with no or ':latest' tag are always pulled") if fledgedNameSpace = os.Getenv("KUBEFLEDGED_NAMESPACE"); fledgedNameSpace == "" { fledgedNameSpace = "kube-fledged" diff --git a/cmd/webhook-server/main.go b/cmd/webhook-server/main.go index 9fce1c43..b28f47ab 100644 --- a/cmd/webhook-server/main.go +++ b/cmd/webhook-server/main.go @@ -262,7 +262,7 @@ func validateImageCache(w http.ResponseWriter, r *http.Request) { } func mutateImageCache(w http.ResponseWriter, r *http.Request) { - serve(w, r, newDelegateToV1AdmitHandler(webhook.MutateImageCache)) + // serve(w, r, newDelegateToV1AdmitHandler(webhook.MutateImageCache)) } func main() { diff --git a/deploy/kubefledged-clusterrolebinding.yaml b/deploy/kubefledged-clusterrolebinding.yaml index c9b6a57b..970f5298 100644 --- a/deploy/kubefledged-clusterrolebinding.yaml +++ b/deploy/kubefledged-clusterrolebinding.yaml @@ -8,7 +8,7 @@ roleRef: name: kubefledged subjects: - kind: ServiceAccount - name: kubefledged + name: kubefledged-controller namespace: kube-fledged - apiGroup: rbac.authorization.k8s.io kind: Group diff --git a/deploy/kubefledged-crd.yaml b/deploy/kubefledged-crd.yaml index 7102b8ac..09ed1cfa 100644 --- a/deploy/kubefledged-crd.yaml +++ b/deploy/kubefledged-crd.yaml @@ -1,9 +1,9 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: - name: imagecaches.fledged.k8s.io + name: imagecaches.kubefledged.k8s.io spec: - group: fledged.k8s.io + group: kubefledged.k8s.io versions: - name: v1alpha1 served: true diff --git a/deploy/kubefledged-deployment-fledged-controller.yaml b/deploy/kubefledged-deployment-controller.yaml similarity index 58% rename from deploy/kubefledged-deployment-fledged-controller.yaml rename to deploy/kubefledged-deployment-controller.yaml index e6575829..3136a122 100644 --- a/deploy/kubefledged-deployment-fledged-controller.yaml +++ b/deploy/kubefledged-deployment-controller.yaml @@ -1,34 +1,32 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: kubefledged + name: kubefledged-controller namespace: kube-fledged spec: replicas: 1 selector: matchLabels: - app: kubefledged + app: kubefledged-controller template: metadata: labels: - app: kubefledged + app: kubefledged-controller spec: containers: - - image: senthilrch/fledged:v0.7.0 - command: ["/opt/bin/fledged"] + - image: senthilrch/kubefledged-controller:v0.7.0 + command: ["/opt/bin/kubefledged-controller"] args: - "--stderrthreshold=INFO" - "--image-pull-deadline-duration=5m" - "--image-cache-refresh-frequency=15m" - - "--docker-client-image=senthilrch/fledged-docker-client:v0.7.0" + - "--cri-client-image=senthilrch/kubefledged-cri-client:v0.7.0" - "--image-pull-policy=IfNotPresent" imagePullPolicy: Always - name: fledged + name: controller env: - name: KUBEFLEDGED_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - serviceAccountName: kubefledged - - + serviceAccountName: kubefledged-controller diff --git a/deploy/kubefledged-deployment-webhook-server.yaml b/deploy/kubefledged-deployment-webhook-server.yaml index 0527cd01..eb99125b 100644 --- a/deploy/kubefledged-deployment-webhook-server.yaml +++ b/deploy/kubefledged-deployment-webhook-server.yaml @@ -14,8 +14,8 @@ spec: app: kubefledged-webhook-server spec: containers: - - image: senthilrch/fledged-webhook-server:v0.7.0 - command: ["/opt/bin/fledged-webhook-server"] + - image: senthilrch/kubefledged-webhook-server:v0.7.0 + command: ["/opt/bin/kubefledged-webhook-server"] args: - "--stderrthreshold=INFO" - "--cert-file=/var/run/secrets/webhook-server/cert.pem" diff --git a/deploy/kubefledged-imagecache.yaml b/deploy/kubefledged-imagecache.yaml index e70b1559..ab945f4f 100644 --- a/deploy/kubefledged-imagecache.yaml +++ b/deploy/kubefledged-imagecache.yaml @@ -1,5 +1,5 @@ --- -apiVersion: fledged.k8s.io/v1alpha1 +apiVersion: kubefledged.k8s.io/v1alpha1 kind: ImageCache metadata: # Name of the image cache. A cluster can have multiple image cache objects diff --git a/deploy/kubefledged-serviceaccount.yaml b/deploy/kubefledged-serviceaccount.yaml index ea7ebbd0..727ed15c 100644 --- a/deploy/kubefledged-serviceaccount.yaml +++ b/deploy/kubefledged-serviceaccount.yaml @@ -1,5 +1,5 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: kubefledged + name: kubefledged-controller namespace: kube-fledged diff --git a/deploy/kubefledged-validatingwebhook.yaml b/deploy/kubefledged-validatingwebhook.yaml index 5e31ef5c..ffb64918 100644 --- a/deploy/kubefledged-validatingwebhook.yaml +++ b/deploy/kubefledged-validatingwebhook.yaml @@ -3,7 +3,7 @@ kind: ValidatingWebhookConfiguration metadata: name: kubefledged webhooks: - - name: validate-image-cache.fledged.k8s.io + - name: validate-image-cache.kubefledged.k8s.io admissionReviewVersions: ["v1"] timeoutSeconds: 1 failurePolicy: Fail @@ -17,7 +17,7 @@ webhooks: caBundle: ${CA_BUNDLE} rules: - operations: ["CREATE", "UPDATE"] - apiGroups: ["fledged.k8s.io"] + apiGroups: ["kubefledged.k8s.io"] apiVersions: ["v1alpha1"] resources: ["imagecaches"] scope: "Namespaced" \ No newline at end of file diff --git a/deploy/webhook-create-signed-cert.sh b/deploy/webhook-create-signed-cert.sh index 5714434e..9e082c9f 100644 --- a/deploy/webhook-create-signed-cert.sh +++ b/deploy/webhook-create-signed-cert.sh @@ -1,5 +1,19 @@ #!/bin/bash +# Copyright 2018 The kube-fledged authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + set -e usage() { diff --git a/deploy/webhook-patch-ca-bundle.sh b/deploy/webhook-patch-ca-bundle.sh index dbb02719..2df0b76c 100644 --- a/deploy/webhook-patch-ca-bundle.sh +++ b/deploy/webhook-patch-ca-bundle.sh @@ -1,5 +1,19 @@ #!/bin/bash +# Copyright 2018 The kube-fledged authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + set -o errexit set -o nounset set -o pipefail diff --git a/pkg/webhook/imagecache.go b/pkg/webhook/imagecache.go index 9f558dbf..e0000783 100644 --- a/pkg/webhook/imagecache.go +++ b/pkg/webhook/imagecache.go @@ -37,6 +37,7 @@ const ( ) // MutateImageCache modifies image cache resource +/* func MutateImageCache(ar v1.AdmissionReview) *v1.AdmissionResponse { glog.V(4).Info("mutating custom resource") cr := struct { @@ -66,6 +67,7 @@ func MutateImageCache(ar v1.AdmissionReview) *v1.AdmissionResponse { } return &reviewResponse } +*/ // ValidateImageCache validates image cache resource func ValidateImageCache(ar v1.AdmissionReview) *v1.AdmissionResponse { From 63a7bb762394fc87970c9e163988b4e844e88030 Mon Sep 17 00:00:00 2001 From: bhuvanessr Date: Fri, 29 May 2020 12:19:58 +0530 Subject: [PATCH 16/21] api group changed to kubefledged.k8s.io --- Makefile | 4 +- cmd/controller/app/controller.go | 144 +++++----- cmd/controller/app/controller_test.go | 248 +++++++++--------- deploy/kubefledged-clusterrole.yaml | 4 +- deploy/kubefledged-deployment-controller.yaml | 5 +- ...kubefledged-deployment-webhook-server.yaml | 5 +- .../kubefledged-service-webhook-server.yaml | 2 +- deploy/kubefledged-validatingwebhook.yaml | 2 +- hack/.golint_failures | 3 +- hack/update-codegen.sh | 2 +- pkg/apis/{fledged => kubefledged}/register.go | 5 +- .../{fledged => kubefledged}/v1alpha1/doc.go | 0 .../v1alpha1/register.go | 4 +- .../v1alpha1/types.go | 0 .../v1alpha1/zz_generated.deepcopy.go | 0 pkg/client/clientset/versioned/clientset.go | 2 +- .../versioned/fake/clientset_generated.go | 4 +- .../clientset/versioned/fake/register.go | 2 +- .../clientset/versioned/scheme/register.go | 2 +- .../{fledged => kubefledged}/v1alpha1/doc.go | 0 .../v1alpha1/fake/doc.go | 0 .../v1alpha1/fake/fake_imagecache.go | 2 +- .../v1alpha1/fake/fake_kubefledged_client.go} | 2 +- .../v1alpha1/generated_expansion.go | 0 .../v1alpha1/imagecache.go | 2 +- .../v1alpha1/kubefledged_client.go} | 2 +- .../informers/externalversions/factory.go | 8 +- .../informers/externalversions/generic.go | 2 +- .../{fledged => kubefledged}/interface.go | 4 +- .../v1alpha1/imagecache.go | 8 +- .../v1alpha1/interface.go | 0 .../v1alpha1/expansion_generated.go | 0 .../v1alpha1/imagecache.go | 2 +- pkg/images/image_helpers.go | 2 +- pkg/images/image_manager.go | 2 +- pkg/images/image_manager_test.go | 2 +- pkg/webhook/imagecache.go | 2 +- 37 files changed, 240 insertions(+), 238 deletions(-) rename pkg/apis/{fledged => kubefledged}/register.go (84%) rename pkg/apis/{fledged => kubefledged}/v1alpha1/doc.go (100%) rename pkg/apis/{fledged => kubefledged}/v1alpha1/register.go (90%) rename pkg/apis/{fledged => kubefledged}/v1alpha1/types.go (100%) rename pkg/apis/{fledged => kubefledged}/v1alpha1/zz_generated.deepcopy.go (100%) rename pkg/client/clientset/versioned/typed/{fledged => kubefledged}/v1alpha1/doc.go (100%) rename pkg/client/clientset/versioned/typed/{fledged => kubefledged}/v1alpha1/fake/doc.go (100%) rename pkg/client/clientset/versioned/typed/{fledged => kubefledged}/v1alpha1/fake/fake_imagecache.go (98%) rename pkg/client/clientset/versioned/typed/{fledged/v1alpha1/fake/fake_fledged_client.go => kubefledged/v1alpha1/fake/fake_kubefledged_client.go} (96%) rename pkg/client/clientset/versioned/typed/{fledged => kubefledged}/v1alpha1/generated_expansion.go (100%) rename pkg/client/clientset/versioned/typed/{fledged => kubefledged}/v1alpha1/imagecache.go (98%) rename pkg/client/clientset/versioned/typed/{fledged/v1alpha1/fledged_client.go => kubefledged/v1alpha1/kubefledged_client.go} (96%) rename pkg/client/informers/externalversions/{fledged => kubefledged}/interface.go (96%) rename pkg/client/informers/externalversions/{fledged => kubefledged}/v1alpha1/imagecache.go (93%) rename pkg/client/informers/externalversions/{fledged => kubefledged}/v1alpha1/interface.go (100%) rename pkg/client/listers/{fledged => kubefledged}/v1alpha1/expansion_generated.go (100%) rename pkg/client/listers/{fledged => kubefledged}/v1alpha1/imagecache.go (97%) diff --git a/Makefile b/Makefile index 82ba7916..cac098cf 100644 --- a/Makefile +++ b/Makefile @@ -193,9 +193,9 @@ deploy-using-operator: kubectl create -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml update: - kubectl scale deployment kubefledged --replicas=0 -n kube-fledged && \ + kubectl scale deployment kubefledged-controller --replicas=0 -n kube-fledged && \ kubectl scale deployment kubefledged-webhook-server --replicas=0 -n kube-fledged && sleep 1 && \ - kubectl scale deployment kubefledged --replicas=1 -n kube-fledged && sleep 1 && \ + kubectl scale deployment kubefledged-controller --replicas=1 -n kube-fledged && sleep 1 && \ kubectl scale deployment kubefledged-webhook-server --replicas=1 -n kube-fledged && sleep 1 && \ kubectl get pods -l app=kubefledged -n kube-fledged diff --git a/cmd/controller/app/controller.go b/cmd/controller/app/controller.go index d9f891d0..a0d07d80 100644 --- a/cmd/controller/app/controller.go +++ b/cmd/controller/app/controller.go @@ -22,11 +22,11 @@ import ( "time" "github.com/golang/glog" - fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + v1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/kubefledged/v1alpha1" clientset "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned" fledgedscheme "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/scheme" - informers "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions/fledged/v1alpha1" - listers "github.com/senthilrch/kube-fledged/pkg/client/listers/fledged/v1alpha1" + informers "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions/kubefledged/v1alpha1" + listers "github.com/senthilrch/kube-fledged/pkg/client/listers/kubefledged/v1alpha1" "github.com/senthilrch/kube-fledged/pkg/images" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -44,10 +44,10 @@ import ( "k8s.io/client-go/util/workqueue" ) -const controllerAgentName = "fledged" -const fledgedCacheSpecValidationKey = "fledged.k8s.io/cachespecvalidation" -const imageCachePurgeAnnotationKey = "fledged.k8s.io/purge-imagecache" -const imageCacheRefreshAnnotationKey = "fledged.k8s.io/refresh-imagecache" +const controllerAgentName = "kubefledged-controller" +const fledgedCacheSpecValidationKey = "kubefledged.k8s.io/cachespecvalidation" +const imageCachePurgeAnnotationKey = "kubefledged.k8s.io/purge-imagecache" +const imageCacheRefreshAnnotationKey = "kubefledged.k8s.io/refresh-imagecache" const ( // SuccessSynced is used as part of the Event 'reason' when a ImageCache is synced @@ -61,8 +61,8 @@ const ( type Controller struct { // kubeclientset is a standard kubernetes clientset kubeclientset kubernetes.Interface - // fledgedclientset is a clientset for fledged.k8s.io API group - fledgedclientset clientset.Interface + // kubefledgedclientset is a clientset for kubefledged.k8s.io API group + kubefledgedclientset clientset.Interface fledgedNameSpace string nodesLister corelisters.NodeLister @@ -87,7 +87,7 @@ type Controller struct { // NewController returns a new fledged controller func NewController( kubeclientset kubernetes.Interface, - fledgedclientset clientset.Interface, + kubefledgedclientset clientset.Interface, namespace string, nodeInformer coreinformers.NodeInformer, imageCacheInformer informers.ImageCacheInformer, @@ -105,7 +105,7 @@ func NewController( controller := &Controller{ kubeclientset: kubeclientset, - fledgedclientset: fledgedclientset, + kubefledgedclientset: kubefledgedclientset, fledgedNameSpace: namespace, nodesLister: nodeInformer.Lister(), nodesSynced: nodeInformer.Informer().HasSynced, @@ -176,7 +176,7 @@ func (c *Controller) danglingJobs() error { // image caches will get refreshed in the next cycle func (c *Controller) danglingImageCaches() error { dangling := false - imagecachelist, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(c.fledgedNameSpace).List(metav1.ListOptions{}) + imagecachelist, err := c.kubefledgedclientset.FledgedV1alpha1().ImageCaches(c.fledgedNameSpace).List(metav1.ListOptions{}) if err != nil { glog.Errorf("Error listing imagecaches: %v", err) return err @@ -186,22 +186,22 @@ func (c *Controller) danglingImageCaches() error { glog.Info("No dangling or stuck imagecaches found...") return nil } - status := &fledgedv1alpha1.ImageCacheStatus{ - Failures: map[string]fledgedv1alpha1.NodeReasonMessageList{}, - Status: fledgedv1alpha1.ImageCacheActionStatusAborted, - Reason: fledgedv1alpha1.ImageCacheReasonImagePullAborted, - Message: fledgedv1alpha1.ImageCacheMessageImagePullAborted, + status := &v1alpha1.ImageCacheStatus{ + Failures: map[string]v1alpha1.NodeReasonMessageList{}, + Status: v1alpha1.ImageCacheActionStatusAborted, + Reason: v1alpha1.ImageCacheReasonImagePullAborted, + Message: v1alpha1.ImageCacheMessageImagePullAborted, } for _, imagecache := range imagecachelist.Items { - if imagecache.Status.Status == fledgedv1alpha1.ImageCacheActionStatusProcessing { + if imagecache.Status.Status == v1alpha1.ImageCacheActionStatusProcessing { status.StartTime = imagecache.Status.StartTime err := c.updateImageCacheStatus(&imagecache, status) if err != nil { - glog.Errorf("Error updating ImageCache(%s) status to '%s': %v", imagecache.Name, fledgedv1alpha1.ImageCacheActionStatusAborted, err) + glog.Errorf("Error updating ImageCache(%s) status to '%s': %v", imagecache.Name, v1alpha1.ImageCacheActionStatusAborted, err) return err } dangling = true - glog.Infof("Dangling Image cache(%s) status changed to '%s'", imagecache.Name, fledgedv1alpha1.ImageCacheActionStatusAborted) + glog.Infof("Dangling Image cache(%s) status changed to '%s'", imagecache.Name, v1alpha1.ImageCacheActionStatusAborted) } } @@ -264,18 +264,18 @@ func (c *Controller) enqueueImageCache(workType images.WorkType, old, new interf switch workType { case images.ImageCacheCreate: obj = new - newImageCache := new.(*fledgedv1alpha1.ImageCache) + newImageCache := new.(*v1alpha1.ImageCache) // If the ImageCache resource already has a status field, it means it's already // synced, so do not queue it for processing - if !reflect.DeepEqual(newImageCache.Status, fledgedv1alpha1.ImageCacheStatus{}) { + if !reflect.DeepEqual(newImageCache.Status, v1alpha1.ImageCacheStatus{}) { return false } case images.ImageCacheUpdate: obj = new - oldImageCache := old.(*fledgedv1alpha1.ImageCache) - newImageCache := new.(*fledgedv1alpha1.ImageCache) + oldImageCache := old.(*v1alpha1.ImageCache) + newImageCache := new.(*v1alpha1.ImageCache) - if oldImageCache.Status.Status == fledgedv1alpha1.ImageCacheActionStatusProcessing { + if oldImageCache.Status.Status == v1alpha1.ImageCacheActionStatusProcessing { if !reflect.DeepEqual(newImageCache.Spec, oldImageCache.Spec) { glog.Warningf("Received image cache update/purge/delete for '%s' while it is under processing, so ignoring.", oldImageCache.Name) return false @@ -319,7 +319,7 @@ func (c *Controller) enqueueImageCache(workType images.WorkType, old, new interf wqKey.WorkType = workType wqKey.ObjKey = key if workType == images.ImageCacheUpdate { - oldImageCache := old.(*fledgedv1alpha1.ImageCache) + oldImageCache := old.(*v1alpha1.ImageCache) wqKey.OldImageCache = oldImageCache } @@ -401,20 +401,20 @@ func (c *Controller) runRefreshWorker() { } for i := range imageCaches { // Do not refresh if status is not yet updated - if reflect.DeepEqual(imageCaches[i].Status, fledgedv1alpha1.ImageCacheStatus{}) { + if reflect.DeepEqual(imageCaches[i].Status, v1alpha1.ImageCacheStatus{}) { continue } // Do not refresh if image cache is already under processing - if imageCaches[i].Status.Status == fledgedv1alpha1.ImageCacheActionStatusProcessing { + if imageCaches[i].Status.Status == v1alpha1.ImageCacheActionStatusProcessing { continue } // Do not refresh image cache if cache spec validation failed - if imageCaches[i].Status.Status == fledgedv1alpha1.ImageCacheActionStatusFailed && - imageCaches[i].Status.Reason == fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed { + if imageCaches[i].Status.Status == v1alpha1.ImageCacheActionStatusFailed && + imageCaches[i].Status.Reason == v1alpha1.ImageCacheReasonCacheSpecValidationFailed { continue } // Do not refresh if image cache has been purged - if imageCaches[i].Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCachePurge { + if imageCaches[i].Status.Reason == v1alpha1.ImageCacheReasonImageCachePurge { continue } c.enqueueImageCache(images.ImageCacheRefresh, imageCaches[i], nil) @@ -425,8 +425,8 @@ func (c *Controller) runRefreshWorker() { // converge the two. It then updates the Status block of the ImageCache resource // with the current status of the resource. func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error { - status := &fledgedv1alpha1.ImageCacheStatus{ - Failures: map[string]fledgedv1alpha1.NodeReasonMessageList{}, + status := &v1alpha1.ImageCacheStatus{ + Failures: map[string]v1alpha1.NodeReasonMessageList{}, } // Convert the namespace/name string into a distinct namespace and name @@ -453,45 +453,45 @@ func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error { } if wqKey.WorkType == images.ImageCacheUpdate && wqKey.OldImageCache == nil { - status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed - status.Reason = fledgedv1alpha1.ImageCacheReasonOldImageCacheNotFound - status.Message = fledgedv1alpha1.ImageCacheMessageOldImageCacheNotFound + status.Status = v1alpha1.ImageCacheActionStatusFailed + status.Reason = v1alpha1.ImageCacheReasonOldImageCacheNotFound + status.Message = v1alpha1.ImageCacheMessageOldImageCacheNotFound if err := c.updateImageCacheStatus(imageCache, status); err != nil { glog.Errorf("Error updating imagecache status to %s: %v", status.Status, err) return err } - glog.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonOldImageCacheNotFound, fledgedv1alpha1.ImageCacheMessageOldImageCacheNotFound) - return fmt.Errorf("%s: %s", fledgedv1alpha1.ImageCacheReasonOldImageCacheNotFound, fledgedv1alpha1.ImageCacheMessageOldImageCacheNotFound) + glog.Errorf("%s: %s", v1alpha1.ImageCacheReasonOldImageCacheNotFound, v1alpha1.ImageCacheMessageOldImageCacheNotFound) + return fmt.Errorf("%s: %s", v1alpha1.ImageCacheReasonOldImageCacheNotFound, v1alpha1.ImageCacheMessageOldImageCacheNotFound) } cacheSpec := imageCache.Spec.CacheSpec glog.V(4).Infof("cacheSpec: %+v", cacheSpec) var nodes []*corev1.Node - status.Status = fledgedv1alpha1.ImageCacheActionStatusProcessing + status.Status = v1alpha1.ImageCacheActionStatusProcessing if wqKey.WorkType == images.ImageCacheCreate { - status.Reason = fledgedv1alpha1.ImageCacheReasonImageCacheCreate - status.Message = fledgedv1alpha1.ImageCacheMessagePullingImages + status.Reason = v1alpha1.ImageCacheReasonImageCacheCreate + status.Message = v1alpha1.ImageCacheMessagePullingImages } if wqKey.WorkType == images.ImageCacheUpdate { - status.Reason = fledgedv1alpha1.ImageCacheReasonImageCacheUpdate - status.Message = fledgedv1alpha1.ImageCacheMessageUpdatingCache + status.Reason = v1alpha1.ImageCacheReasonImageCacheUpdate + status.Message = v1alpha1.ImageCacheMessageUpdatingCache } if wqKey.WorkType == images.ImageCacheRefresh { - status.Reason = fledgedv1alpha1.ImageCacheReasonImageCacheRefresh - status.Message = fledgedv1alpha1.ImageCacheMessageRefreshingCache + status.Reason = v1alpha1.ImageCacheReasonImageCacheRefresh + status.Message = v1alpha1.ImageCacheMessageRefreshingCache } if wqKey.WorkType == images.ImageCachePurge { - status.Reason = fledgedv1alpha1.ImageCacheReasonImageCachePurge - status.Message = fledgedv1alpha1.ImageCacheMessagePurgeCache + status.Reason = v1alpha1.ImageCacheReasonImageCachePurge + status.Message = v1alpha1.ImageCacheMessagePurgeCache } - imageCache, err = c.fledgedclientset.FledgedV1alpha1().ImageCaches(namespace).Get(name, metav1.GetOptions{}) + imageCache, err = c.kubefledgedclientset.FledgedV1alpha1().ImageCaches(namespace).Get(name, metav1.GetOptions{}) if err != nil { glog.Errorf("Error getting imagecache(%s) from api server: %v", name, err) return err @@ -564,7 +564,7 @@ func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error { // Finally, we update the status block of the ImageCache resource to reflect the // current state of the world // Get the ImageCache resource with this namespace/name - imageCache, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(namespace).Get(name, metav1.GetOptions{}) + imageCache, err := c.kubefledgedclientset.FledgedV1alpha1().ImageCaches(namespace).Get(name, metav1.GetOptions{}) if err != nil { glog.Errorf("Error getting image cache %s: %v", name, err) return err @@ -579,25 +579,25 @@ func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error { failures := false for _, v := range *wqKey.Status { if (v.Status == images.ImageWorkResultStatusSucceeded || v.Status == images.ImageWorkResultStatusAlreadyPulled) && !failures { - status.Status = fledgedv1alpha1.ImageCacheActionStatusSucceeded + status.Status = v1alpha1.ImageCacheActionStatusSucceeded if v.ImageWorkRequest.WorkType == images.ImageCachePurge { - status.Message = fledgedv1alpha1.ImageCacheMessageImagesDeletedSuccessfully + status.Message = v1alpha1.ImageCacheMessageImagesDeletedSuccessfully } else { - status.Message = fledgedv1alpha1.ImageCacheMessageImagesPulledSuccessfully + status.Message = v1alpha1.ImageCacheMessageImagesPulledSuccessfully } } if v.Status == images.ImageWorkResultStatusFailed && !failures { failures = true - status.Status = fledgedv1alpha1.ImageCacheActionStatusFailed + status.Status = v1alpha1.ImageCacheActionStatusFailed if v.ImageWorkRequest.WorkType == images.ImageCachePurge { - status.Message = fledgedv1alpha1.ImageCacheMessageImageDeleteFailedForSomeImages + status.Message = v1alpha1.ImageCacheMessageImageDeleteFailedForSomeImages } else { - status.Message = fledgedv1alpha1.ImageCacheMessageImagePullFailedForSomeImages + status.Message = v1alpha1.ImageCacheMessageImagePullFailedForSomeImages } } if v.Status == images.ImageWorkResultStatusFailed { status.Failures[v.ImageWorkRequest.Image] = append( - status.Failures[v.ImageWorkRequest.Image], fledgedv1alpha1.NodeReasonMessage{ + status.Failures[v.ImageWorkRequest.Image], v1alpha1.NodeReasonMessage{ Node: v.ImageWorkRequest.Node.Labels["kubernetes.io/hostname"], Reason: v.Reason, Message: v.Message, @@ -611,19 +611,19 @@ func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error { return err } - if imageCache.Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCachePurge || imageCache.Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCacheRefresh { - imageCache, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(namespace).Get(name, metav1.GetOptions{}) + if imageCache.Status.Reason == v1alpha1.ImageCacheReasonImageCachePurge || imageCache.Status.Reason == v1alpha1.ImageCacheReasonImageCacheRefresh { + imageCache, err := c.kubefledgedclientset.FledgedV1alpha1().ImageCaches(namespace).Get(name, metav1.GetOptions{}) if err != nil { glog.Errorf("Error getting image cache %s: %v", name, err) return err } - if imageCache.Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCachePurge { + if imageCache.Status.Reason == v1alpha1.ImageCacheReasonImageCachePurge { if err := c.removeAnnotation(imageCache, imageCachePurgeAnnotationKey); err != nil { glog.Errorf("Error removing Annotation %s from imagecache(%s): %v", imageCachePurgeAnnotationKey, imageCache.Name, err) return err } } - if imageCache.Status.Reason == fledgedv1alpha1.ImageCacheReasonImageCacheRefresh { + if imageCache.Status.Reason == v1alpha1.ImageCacheReasonImageCacheRefresh { if err := c.removeAnnotation(imageCache, imageCacheRefreshAnnotationKey); err != nil { glog.Errorf("Error removing Annotation %s from imagecache(%s): %v", imageCacheRefreshAnnotationKey, imageCache.Name, err) return err @@ -631,11 +631,11 @@ func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error { } } - if status.Status == fledgedv1alpha1.ImageCacheActionStatusSucceeded { + if status.Status == v1alpha1.ImageCacheActionStatusSucceeded { c.recorder.Event(imageCache, corev1.EventTypeNormal, status.Reason, status.Message) } - if status.Status == fledgedv1alpha1.ImageCacheActionStatusFailed { + if status.Status == v1alpha1.ImageCacheActionStatusFailed { c.recorder.Event(imageCache, corev1.EventTypeWarning, status.Reason, status.Message) } } @@ -644,13 +644,13 @@ func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error { } -func (c *Controller) updateImageCacheStatus(imageCache *fledgedv1alpha1.ImageCache, status *fledgedv1alpha1.ImageCacheStatus) error { +func (c *Controller) updateImageCacheStatus(imageCache *v1alpha1.ImageCache, status *v1alpha1.ImageCacheStatus) error { // NEVER modify objects from the store. It's a read-only, local cache. // You can use DeepCopy() to make a deep copy of original object and modify this copy // Or create a copy manually for better performance imageCacheCopy := imageCache.DeepCopy() imageCacheCopy.Status = *status - if imageCacheCopy.Status.Status != fledgedv1alpha1.ImageCacheActionStatusProcessing { + if imageCacheCopy.Status.Status != v1alpha1.ImageCacheActionStatusProcessing { completionTime := metav1.Now() imageCacheCopy.Status.CompletionTime = &completionTime } @@ -658,11 +658,11 @@ func (c *Controller) updateImageCacheStatus(imageCache *fledgedv1alpha1.ImageCac // we must use Update instead of UpdateStatus to update the Status block of the ImageCache resource. // UpdateStatus will not allow changes to the Spec of the resource, // which is ideal for ensuring nothing other than resource status has been updated. - _, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(imageCache.Namespace).Update(imageCacheCopy) + _, err := c.kubefledgedclientset.FledgedV1alpha1().ImageCaches(imageCache.Namespace).Update(imageCacheCopy) return err } -func (c *Controller) updateImageCacheSpecAndStatus(imageCache *fledgedv1alpha1.ImageCache, spec fledgedv1alpha1.ImageCacheSpec, status *fledgedv1alpha1.ImageCacheStatus) error { +func (c *Controller) updateImageCacheSpecAndStatus(imageCache *v1alpha1.ImageCache, spec v1alpha1.ImageCacheSpec, status *v1alpha1.ImageCacheStatus) error { // NEVER modify objects from the store. It's a read-only, local cache. // You can use DeepCopy() to make a deep copy of original object and modify this copy // Or create a copy manually for better performance @@ -670,13 +670,13 @@ func (c *Controller) updateImageCacheSpecAndStatus(imageCache *fledgedv1alpha1.I imageCacheCopy.Spec = spec imageCacheCopy.Status = *status - if status.Status == fledgedv1alpha1.ImageCacheActionStatusFailed && - status.Reason == fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed { + if status.Status == v1alpha1.ImageCacheActionStatusFailed && + status.Reason == v1alpha1.ImageCacheReasonCacheSpecValidationFailed { imageCacheCopy.Annotations = make(map[string]string) imageCacheCopy.Annotations[fledgedCacheSpecValidationKey] = "failed" } - if imageCacheCopy.Status.Status != fledgedv1alpha1.ImageCacheActionStatusProcessing { + if imageCacheCopy.Status.Status != v1alpha1.ImageCacheActionStatusProcessing { completionTime := metav1.Now() imageCacheCopy.Status.CompletionTime = &completionTime } @@ -684,14 +684,14 @@ func (c *Controller) updateImageCacheSpecAndStatus(imageCache *fledgedv1alpha1.I // we must use Update instead of UpdateStatus to update the Status block of the ImageCache resource. // UpdateStatus will not allow changes to the Spec of the resource, // which is ideal for ensuring nothing other than resource status has been updated. - _, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(imageCache.Namespace).Update(imageCacheCopy) + _, err := c.kubefledgedclientset.FledgedV1alpha1().ImageCaches(imageCache.Namespace).Update(imageCacheCopy) return err } -func (c *Controller) removeAnnotation(imageCache *fledgedv1alpha1.ImageCache, annotationKey string) error { +func (c *Controller) removeAnnotation(imageCache *v1alpha1.ImageCache, annotationKey string) error { imageCacheCopy := imageCache.DeepCopy() delete(imageCacheCopy.Annotations, annotationKey) - _, err := c.fledgedclientset.FledgedV1alpha1().ImageCaches(imageCache.Namespace).Update(imageCacheCopy) + _, err := c.kubefledgedclientset.FledgedV1alpha1().ImageCaches(imageCache.Namespace).Update(imageCacheCopy) if err == nil { glog.Infof("Annotation %s removed from imagecache(%s)", fledgedCacheSpecValidationKey, imageCache.Name) } diff --git a/cmd/controller/app/controller_test.go b/cmd/controller/app/controller_test.go index b6b0a8c5..e2fa5eed 100644 --- a/cmd/controller/app/controller_test.go +++ b/cmd/controller/app/controller_test.go @@ -22,11 +22,11 @@ import ( "testing" "time" - fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + kubefledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/kubefledged/v1alpha1" clientset "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned" - fledgedclientsetfake "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/fake" + kubefledgedclientsetfake "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/fake" informers "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions" - fledgedinformers "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions/fledged/v1alpha1" + kubefledgedinformers "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions/kubefledged/v1alpha1" "github.com/senthilrch/kube-fledged/pkg/images" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -53,7 +53,7 @@ func noResyncPeriodFunc() time.Duration { return 0 } -func newTestController(kubeclientset kubernetes.Interface, fledgedclientset clientset.Interface) (*Controller, coreinformers.NodeInformer, fledgedinformers.ImageCacheInformer) { +func newTestController(kubeclientset kubernetes.Interface, fledgedclientset clientset.Interface) (*Controller, coreinformers.NodeInformer, kubefledgedinformers.ImageCacheInformer) { kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeclientset, noResyncPeriodFunc()) fledgedInformerFactory := informers.NewSharedInformerFactory(fledgedclientset, noResyncPeriodFunc()) nodeInformer := kubeInformerFactory.Core().V1().Nodes() @@ -84,7 +84,7 @@ func TestPreFlightChecks(t *testing.T) { jobList *batchv1.JobList jobListError error jobDeleteError error - imageCacheList *fledgedv1alpha1.ImageCacheList + imageCacheList *kubefledgedv1alpha1.ImageCacheList imageCacheListError error imageCacheUpdateError error expectErr bool @@ -95,7 +95,7 @@ func TestPreFlightChecks(t *testing.T) { jobList: &batchv1.JobList{Items: []batchv1.Job{}}, jobListError: nil, jobDeleteError: nil, - imageCacheList: &fledgedv1alpha1.ImageCacheList{Items: []fledgedv1alpha1.ImageCache{}}, + imageCacheList: &kubefledgedv1alpha1.ImageCacheList{Items: []kubefledgedv1alpha1.ImageCache{}}, imageCacheListError: nil, imageCacheUpdateError: nil, expectErr: false, @@ -106,14 +106,14 @@ func TestPreFlightChecks(t *testing.T) { jobList: &batchv1.JobList{Items: []batchv1.Job{}}, jobListError: nil, jobDeleteError: nil, - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ + imageCacheList: &kubefledgedv1alpha1.ImageCacheList{ + Items: []kubefledgedv1alpha1.ImageCache{ { ObjectMeta: metav1.ObjectMeta{ Name: "foo", }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, + Status: kubefledgedv1alpha1.ImageCacheStatus{ + Status: kubefledgedv1alpha1.ImageCacheActionStatusSucceeded, }, }, }, @@ -137,14 +137,14 @@ func TestPreFlightChecks(t *testing.T) { }, jobListError: nil, jobDeleteError: nil, - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ + imageCacheList: &kubefledgedv1alpha1.ImageCacheList{ + Items: []kubefledgedv1alpha1.ImageCache{ { ObjectMeta: metav1.ObjectMeta{ Name: "foo", }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, + Status: kubefledgedv1alpha1.ImageCacheStatus{ + Status: kubefledgedv1alpha1.ImageCacheActionStatusProcessing, }, }, }, @@ -194,14 +194,14 @@ func TestPreFlightChecks(t *testing.T) { jobList: &batchv1.JobList{Items: []batchv1.Job{}}, jobListError: nil, jobDeleteError: nil, - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ + imageCacheList: &kubefledgedv1alpha1.ImageCacheList{ + Items: []kubefledgedv1alpha1.ImageCache{ { ObjectMeta: metav1.ObjectMeta{ Name: "foo", }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, + Status: kubefledgedv1alpha1.ImageCacheStatus{ + Status: kubefledgedv1alpha1.ImageCacheActionStatusProcessing, }, }, }, @@ -214,7 +214,7 @@ func TestPreFlightChecks(t *testing.T) { } for _, test := range tests { fakekubeclientset := &fakeclientset.Clientset{} - fakefledgedclientset := &fledgedclientsetfake.Clientset{} + fakefledgedclientset := &kubefledgedclientsetfake.Clientset{} if test.jobListError != nil { listError := apierrors.NewInternalError(test.jobListError) fakekubeclientset.AddReactor("list", "jobs", func(action core.Action) (handled bool, ret runtime.Object, err error) { @@ -276,14 +276,14 @@ func TestPreFlightChecks(t *testing.T) { func TestRunRefreshWorker(t *testing.T) { tests := []struct { name string - imageCacheList *fledgedv1alpha1.ImageCacheList + imageCacheList *kubefledgedv1alpha1.ImageCacheList imageCacheListError error workqueueItems int }{ { name: "#1: Do not refresh if status is not yet updated", - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ + imageCacheList: &kubefledgedv1alpha1.ImageCacheList{ + Items: []kubefledgedv1alpha1.ImageCache{ { ObjectMeta: metav1.ObjectMeta{ Name: "foo", @@ -297,15 +297,15 @@ func TestRunRefreshWorker(t *testing.T) { }, { name: "#2: Do not refresh if image cache is already under processing", - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ + imageCacheList: &kubefledgedv1alpha1.ImageCacheList{ + Items: []kubefledgedv1alpha1.ImageCache{ { ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, + Status: kubefledgedv1alpha1.ImageCacheStatus{ + Status: kubefledgedv1alpha1.ImageCacheActionStatusProcessing, }, }, }, @@ -315,16 +315,16 @@ func TestRunRefreshWorker(t *testing.T) { }, { name: "#3: Do not refresh image cache if cache spec validation failed", - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ + imageCacheList: &kubefledgedv1alpha1.ImageCacheList{ + Items: []kubefledgedv1alpha1.ImageCache{ { ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusFailed, - Reason: fledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, + Status: kubefledgedv1alpha1.ImageCacheStatus{ + Status: kubefledgedv1alpha1.ImageCacheActionStatusFailed, + Reason: kubefledgedv1alpha1.ImageCacheReasonCacheSpecValidationFailed, }, }, }, @@ -334,15 +334,15 @@ func TestRunRefreshWorker(t *testing.T) { }, { name: "#4: Do not refresh if image cache has been purged", - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ + imageCacheList: &kubefledgedv1alpha1.ImageCacheList{ + Items: []kubefledgedv1alpha1.ImageCache{ { ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Reason: fledgedv1alpha1.ImageCacheReasonImageCachePurge, + Status: kubefledgedv1alpha1.ImageCacheStatus{ + Reason: kubefledgedv1alpha1.ImageCacheReasonImageCachePurge, }, }, }, @@ -352,15 +352,15 @@ func TestRunRefreshWorker(t *testing.T) { }, { name: "#5: Successfully queued 1 imagecache for refresh", - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ + imageCacheList: &kubefledgedv1alpha1.ImageCacheList{ + Items: []kubefledgedv1alpha1.ImageCache{ { ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, + Status: kubefledgedv1alpha1.ImageCacheStatus{ + Status: kubefledgedv1alpha1.ImageCacheActionStatusSucceeded, }, }, }, @@ -370,15 +370,15 @@ func TestRunRefreshWorker(t *testing.T) { }, { name: "#6: Successfully queued 2 imagecaches for refresh", - imageCacheList: &fledgedv1alpha1.ImageCacheList{ - Items: []fledgedv1alpha1.ImageCache{ + imageCacheList: &kubefledgedv1alpha1.ImageCacheList{ + Items: []kubefledgedv1alpha1.ImageCache{ { ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, + Status: kubefledgedv1alpha1.ImageCacheStatus{ + Status: kubefledgedv1alpha1.ImageCacheActionStatusSucceeded, }, }, { @@ -386,8 +386,8 @@ func TestRunRefreshWorker(t *testing.T) { Name: "bar", Namespace: "kube-fledged", }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusFailed, + Status: kubefledgedv1alpha1.ImageCacheStatus{ + Status: kubefledgedv1alpha1.ImageCacheActionStatusFailed, }, }, }, @@ -409,7 +409,7 @@ func TestRunRefreshWorker(t *testing.T) { continue } fakekubeclientset := &fakeclientset.Clientset{} - fakefledgedclientset := &fledgedclientsetfake.Clientset{} + fakefledgedclientset := &kubefledgedclientsetfake.Clientset{} controller, _, imagecacheInformer := newTestController(fakekubeclientset, fakefledgedclientset) if test.imageCacheList != nil && len(test.imageCacheList.Items) > 0 { @@ -431,13 +431,13 @@ func TestSyncHandler(t *testing.T) { reaction string } now := metav1.Now() - defaultImageCache := fledgedv1alpha1.ImageCache{ + defaultImageCache := kubefledgedv1alpha1.ImageCache{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + Spec: kubefledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []kubefledgedv1alpha1.CacheSpecImages{ { Images: []string{"foo"}, }, @@ -457,7 +457,7 @@ func TestSyncHandler(t *testing.T) { tests := []struct { name string - imageCache fledgedv1alpha1.ImageCache + imageCache kubefledgedv1alpha1.ImageCache wqKey images.WorkQueueKey nodeList *corev1.NodeList expectedActions []ActionReaction @@ -474,13 +474,13 @@ func TestSyncHandler(t *testing.T) { }, /*{ name: "#2: Create - Invalid imagecache spec (no images specified)", - imageCache: fledgedv1alpha1.ImageCache{ + imageCache: kubefledgedv1alpha1.ImageCache{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + Spec: kubefledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []kubefledgedv1alpha1.CacheSpecImages{ { Images: []string{}, }, @@ -514,13 +514,13 @@ func TestSyncHandler(t *testing.T) { wqKey: images.WorkQueueKey{ ObjKey: "kube-fledged/foo", WorkType: images.ImageCacheUpdate, - OldImageCache: &fledgedv1alpha1.ImageCache{ + OldImageCache: &kubefledgedv1alpha1.ImageCache{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + Spec: kubefledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []kubefledgedv1alpha1.CacheSpecImages{ { Images: []string{"foo"}, }, @@ -542,13 +542,13 @@ func TestSyncHandler(t *testing.T) { wqKey: images.WorkQueueKey{ ObjKey: "kube-fledged/foo", WorkType: images.ImageCacheUpdate, - OldImageCache: &fledgedv1alpha1.ImageCache{ + OldImageCache: &kubefledgedv1alpha1.ImageCache{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + Spec: kubefledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []kubefledgedv1alpha1.CacheSpecImages{ { Images: []string{"foo"}, NodeSelector: map[string]string{"foo": "bar"}, @@ -579,22 +579,22 @@ func TestSyncHandler(t *testing.T) { }, { name: "#7: StatusUpdate - Successful Refresh", - imageCache: fledgedv1alpha1.ImageCache{ + imageCache: kubefledgedv1alpha1.ImageCache{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + Spec: kubefledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []kubefledgedv1alpha1.CacheSpecImages{ { Images: []string{"foo"}, }, }, }, - Status: fledgedv1alpha1.ImageCacheStatus{ + Status: kubefledgedv1alpha1.ImageCacheStatus{ StartTime: &now, - Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, - Reason: fledgedv1alpha1.ImageCacheReasonImageCacheRefresh, + Status: kubefledgedv1alpha1.ImageCacheActionStatusProcessing, + Reason: kubefledgedv1alpha1.ImageCacheReasonImageCacheRefresh, }, }, wqKey: images.WorkQueueKey{ @@ -634,20 +634,20 @@ func TestSyncHandler(t *testing.T) { }, { name: "#9: Purge - Successful purge", - imageCache: fledgedv1alpha1.ImageCache{ + imageCache: kubefledgedv1alpha1.ImageCache{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + Spec: kubefledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []kubefledgedv1alpha1.CacheSpecImages{ { Images: []string{"foo"}, }, }, }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Reason: fledgedv1alpha1.ImageCacheReasonImageCachePurge, + Status: kubefledgedv1alpha1.ImageCacheStatus{ + Reason: kubefledgedv1alpha1.ImageCacheReasonImageCachePurge, }, }, wqKey: images.WorkQueueKey{ @@ -683,13 +683,13 @@ func TestSyncHandler(t *testing.T) { wqKey: images.WorkQueueKey{ ObjKey: "kube-fledged/foo", WorkType: images.ImageCacheUpdate, - OldImageCache: &fledgedv1alpha1.ImageCache{ + OldImageCache: &kubefledgedv1alpha1.ImageCache{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + Spec: kubefledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []kubefledgedv1alpha1.CacheSpecImages{ { Images: []string{"foo", "bar"}, }, @@ -707,19 +707,19 @@ func TestSyncHandler(t *testing.T) { }, { name: "#12: StatusUpdate - ImagesPulledSuccessfully", - imageCache: fledgedv1alpha1.ImageCache{ + imageCache: kubefledgedv1alpha1.ImageCache{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + Spec: kubefledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []kubefledgedv1alpha1.CacheSpecImages{ { Images: []string{"foo"}, }, }, }, - Status: fledgedv1alpha1.ImageCacheStatus{ + Status: kubefledgedv1alpha1.ImageCacheStatus{ StartTime: &now, }, }, @@ -745,22 +745,22 @@ func TestSyncHandler(t *testing.T) { }, { name: "#13: StatusUpdate - ImagesDeletedSuccessfully", - imageCache: fledgedv1alpha1.ImageCache{ + imageCache: kubefledgedv1alpha1.ImageCache{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + Spec: kubefledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []kubefledgedv1alpha1.CacheSpecImages{ { Images: []string{"foo"}, }, }, }, - Status: fledgedv1alpha1.ImageCacheStatus{ + Status: kubefledgedv1alpha1.ImageCacheStatus{ StartTime: &now, - Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, - Reason: fledgedv1alpha1.ImageCacheReasonImageCachePurge, + Status: kubefledgedv1alpha1.ImageCacheActionStatusProcessing, + Reason: kubefledgedv1alpha1.ImageCacheReasonImageCachePurge, }, }, wqKey: images.WorkQueueKey{ @@ -833,7 +833,7 @@ func TestSyncHandler(t *testing.T) { for _, test := range tests { fakekubeclientset := &fakeclientset.Clientset{} - fakefledgedclientset := &fledgedclientsetfake.Clientset{} + fakefledgedclientset := &kubefledgedclientsetfake.Clientset{} for _, ar := range test.expectedActions { if ar.reaction != "" { apiError := apierrors.NewInternalError(fmt.Errorf(ar.reaction)) @@ -871,13 +871,13 @@ func TestSyncHandler(t *testing.T) { func TestEnqueueImageCache(t *testing.T) { //now := metav1.Now() //nowplus5s := metav1.NewTime(time.Now().Add(time.Second * 5)) - defaultImageCache := fledgedv1alpha1.ImageCache{ + defaultImageCache := kubefledgedv1alpha1.ImageCache{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + Spec: kubefledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []kubefledgedv1alpha1.CacheSpecImages{ { Images: []string{"foo"}, }, @@ -887,8 +887,8 @@ func TestEnqueueImageCache(t *testing.T) { tests := []struct { name string workType images.WorkType - oldImageCache fledgedv1alpha1.ImageCache - newImageCache fledgedv1alpha1.ImageCache + oldImageCache kubefledgedv1alpha1.ImageCache + newImageCache kubefledgedv1alpha1.ImageCache expectedResult bool }{ { @@ -900,20 +900,20 @@ func TestEnqueueImageCache(t *testing.T) { { name: "#2: Create - Imagecache with Status field, so no queueing", workType: images.ImageCacheCreate, - newImageCache: fledgedv1alpha1.ImageCache{ + newImageCache: kubefledgedv1alpha1.ImageCache{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + Spec: kubefledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []kubefledgedv1alpha1.CacheSpecImages{ { Images: []string{"foo"}, }, }, }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, + Status: kubefledgedv1alpha1.ImageCacheStatus{ + Status: kubefledgedv1alpha1.ImageCacheActionStatusSucceeded, }, }, expectedResult: false, @@ -922,21 +922,21 @@ func TestEnqueueImageCache(t *testing.T) { name: "#3: Update - Imagecache purge. Successful queueing", workType: images.ImageCacheUpdate, oldImageCache: defaultImageCache, - newImageCache: fledgedv1alpha1.ImageCache{ + newImageCache: kubefledgedv1alpha1.ImageCache{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", Annotations: map[string]string{imageCachePurgeAnnotationKey: ""}, }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + Spec: kubefledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []kubefledgedv1alpha1.CacheSpecImages{ { Images: []string{"foo"}, }, }, }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, + Status: kubefledgedv1alpha1.ImageCacheStatus{ + Status: kubefledgedv1alpha1.ImageCacheActionStatusSucceeded, }, }, expectedResult: true, @@ -951,20 +951,20 @@ func TestEnqueueImageCache(t *testing.T) { { name: "#5: Update - Status processing. Unsuccessful queueing", workType: images.ImageCacheUpdate, - oldImageCache: fledgedv1alpha1.ImageCache{ + oldImageCache: kubefledgedv1alpha1.ImageCache{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + Spec: kubefledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []kubefledgedv1alpha1.CacheSpecImages{ { Images: []string{"foo"}, }, }, }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusProcessing, + Status: kubefledgedv1alpha1.ImageCacheStatus{ + Status: kubefledgedv1alpha1.ImageCacheActionStatusProcessing, }, }, expectedResult: false, @@ -973,13 +973,13 @@ func TestEnqueueImageCache(t *testing.T) { name: "#6: Update - Successful queueing", workType: images.ImageCacheUpdate, oldImageCache: defaultImageCache, - newImageCache: fledgedv1alpha1.ImageCache{ + newImageCache: kubefledgedv1alpha1.ImageCache{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + Spec: kubefledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []kubefledgedv1alpha1.CacheSpecImages{ { Images: []string{"foo", "bar"}, }, @@ -1003,7 +1003,7 @@ func TestEnqueueImageCache(t *testing.T) { name: "#9: Update - CacheSpec restoration", workType: images.ImageCacheUpdate, oldImageCache: defaultImageCache, - newImageCache: fledgedv1alpha1.ImageCache{ + newImageCache: kubefledgedv1alpha1.ImageCache{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", @@ -1011,8 +1011,8 @@ func TestEnqueueImageCache(t *testing.T) { fledgedCacheSpecValidationKey: "failed", }, }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + Spec: kubefledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []kubefledgedv1alpha1.CacheSpecImages{ { Images: []string{"foo", "bar"}, }, @@ -1025,21 +1025,21 @@ func TestEnqueueImageCache(t *testing.T) { name: "#10: Update - Imagecache refresh. Successful queueing", workType: images.ImageCacheUpdate, oldImageCache: defaultImageCache, - newImageCache: fledgedv1alpha1.ImageCache{ + newImageCache: kubefledgedv1alpha1.ImageCache{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", Annotations: map[string]string{imageCacheRefreshAnnotationKey: ""}, }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + Spec: kubefledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []kubefledgedv1alpha1.CacheSpecImages{ { Images: []string{"foo"}, }, }, }, - Status: fledgedv1alpha1.ImageCacheStatus{ - Status: fledgedv1alpha1.ImageCacheActionStatusSucceeded, + Status: kubefledgedv1alpha1.ImageCacheStatus{ + Status: kubefledgedv1alpha1.ImageCacheActionStatusSucceeded, }, }, expectedResult: true, @@ -1048,7 +1048,7 @@ func TestEnqueueImageCache(t *testing.T) { for _, test := range tests { fakekubeclientset := &fakeclientset.Clientset{} - fakefledgedclientset := &fledgedclientsetfake.Clientset{} + fakefledgedclientset := &kubefledgedclientsetfake.Clientset{} controller, _, _ := newTestController(fakekubeclientset, fakefledgedclientset) result := controller.enqueueImageCache(test.workType, &test.oldImageCache, &test.newImageCache) if result != test.expectedResult { @@ -1062,13 +1062,13 @@ func TestProcessNextWorkItem(t *testing.T) { action string reaction string } - defaultImageCache := fledgedv1alpha1.ImageCache{ + defaultImageCache := kubefledgedv1alpha1.ImageCache{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + Spec: kubefledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []kubefledgedv1alpha1.CacheSpecImages{ { Images: []string{"foo"}, }, @@ -1078,7 +1078,7 @@ func TestProcessNextWorkItem(t *testing.T) { tests := []struct { name string - imageCache fledgedv1alpha1.ImageCache + imageCache kubefledgedv1alpha1.ImageCache wqKey images.WorkQueueKey expectedActions []ActionReaction expectErr bool @@ -1109,13 +1109,13 @@ func TestProcessNextWorkItem(t *testing.T) { }, { name: "#2: Create - Invalid imagecache spec (no images specified)", - imageCache: fledgedv1alpha1.ImageCache{ + imageCache: kubefledgedv1alpha1.ImageCache{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "kube-fledged", }, - Spec: fledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []fledgedv1alpha1.CacheSpecImages{ + Spec: kubefledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []kubefledgedv1alpha1.CacheSpecImages{ { Images: []string{}, }, @@ -1139,7 +1139,7 @@ func TestProcessNextWorkItem(t *testing.T) { for _, test := range tests { fakekubeclientset := &fakeclientset.Clientset{} - fakefledgedclientset := &fledgedclientsetfake.Clientset{} + fakefledgedclientset := &kubefledgedclientsetfake.Clientset{} for _, ar := range test.expectedActions { if ar.reaction != "" { apiError := apierrors.NewInternalError(fmt.Errorf(ar.reaction)) diff --git a/deploy/kubefledged-clusterrole.yaml b/deploy/kubefledged-clusterrole.yaml index 9a39a6f0..4c8f3f10 100644 --- a/deploy/kubefledged-clusterrole.yaml +++ b/deploy/kubefledged-clusterrole.yaml @@ -6,7 +6,7 @@ metadata: rbac.authorization.kubernetes.io/autoupdate: "true" rules: - apiGroups: - - "fledged.k8s.io" + - "kubefledged.k8s.io" resources: - imagecaches verbs: @@ -15,7 +15,7 @@ rules: - watch - update - apiGroups: - - "fledged.k8s.io" + - "kubefledged.k8s.io" resources: - imagecaches/status verbs: diff --git a/deploy/kubefledged-deployment-controller.yaml b/deploy/kubefledged-deployment-controller.yaml index 3136a122..aed33df3 100644 --- a/deploy/kubefledged-deployment-controller.yaml +++ b/deploy/kubefledged-deployment-controller.yaml @@ -7,11 +7,12 @@ spec: replicas: 1 selector: matchLabels: - app: kubefledged-controller + kubefledged: kubefledged-controller template: metadata: labels: - app: kubefledged-controller + kubefledged: kubefledged-controller + app: kubefledged spec: containers: - image: senthilrch/kubefledged-controller:v0.7.0 diff --git a/deploy/kubefledged-deployment-webhook-server.yaml b/deploy/kubefledged-deployment-webhook-server.yaml index eb99125b..f9f3d5ef 100644 --- a/deploy/kubefledged-deployment-webhook-server.yaml +++ b/deploy/kubefledged-deployment-webhook-server.yaml @@ -7,11 +7,12 @@ spec: replicas: 1 selector: matchLabels: - app: kubefledged-webhook-server + kubefledged: kubefledged-webhook-server template: metadata: labels: - app: kubefledged-webhook-server + kubefledged: kubefledged-webhook-server + app: kubefledged spec: containers: - image: senthilrch/kubefledged-webhook-server:v0.7.0 diff --git a/deploy/kubefledged-service-webhook-server.yaml b/deploy/kubefledged-service-webhook-server.yaml index c68a4ac7..d6487b2a 100644 --- a/deploy/kubefledged-service-webhook-server.yaml +++ b/deploy/kubefledged-service-webhook-server.yaml @@ -10,5 +10,5 @@ spec: protocol: TCP targetPort: 443 selector: - app: kubefledged-webhook-server + kubefledged: kubefledged-webhook-server type: ClusterIP diff --git a/deploy/kubefledged-validatingwebhook.yaml b/deploy/kubefledged-validatingwebhook.yaml index ffb64918..d0602abc 100644 --- a/deploy/kubefledged-validatingwebhook.yaml +++ b/deploy/kubefledged-validatingwebhook.yaml @@ -14,7 +14,7 @@ webhooks: name: kubefledged-webhook-server path: "/validate-image-cache" port: 3443 - caBundle: ${CA_BUNDLE} + caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURERENDQWZTZ0F3SUJBZ0lSQU52ZlM5L0tWK3hJeEZ3blZneVNzVkl3RFFZSktvWklodmNOQVFFTEJRQXcKTHpFdE1Dc0dBMVVFQXhNa05HRXhaREUyTmprdE9XUmpaQzAwWmpoa0xXRTJPR1V0WVRoaFlXUXhNVGRtT1dSagpNQjRYRFRJd01EVXlPVEEwTlRNeE9Wb1hEVEkxTURVeU9EQTFOVE14T1Zvd0x6RXRNQ3NHQTFVRUF4TWtOR0V4ClpERTJOamt0T1dSalpDMDBaamhrTFdFMk9HVXRZVGhoWVdReE1UZG1PV1JqTUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDRNakpWQUFYakdwVkxnWmt2cEVCOEFubEdMaERHakNHYXQ5eHk0VApoc0JlbGVTeldkUlZHZVBGTW9HNERMMndGQmRoNThHUmlxNGtNMy9LdnVMNld3c3dRYzRYWHJXYXYwdjAzYVdIClFvbWRTUVFlSnJncXlYL3dxanFYVkZ2bGtoejR1UjRaaHNoekxiMEFpLzg2Vnc5cUI2cXZkNW9tbDZBSUJZVEkKTEVQcnJTbjlMb0NEb3JNMDQ5a3AxbldDMXY0OHlmSlJ5RUN4aTRGOVRZRnE1ZDcyU09tMGNtNVZDN3A5bllBVQpvTG5VWkRkc1BLdWFHNXZHaU5GSDk1VTM2aXNsekRTV09YaFZORi9TcUxOeGJDWGZsS0wwRzZLdi9YTllGc2NPCjhaR1ArZWdBSHNjMG1VMkhCc2NDcDcxRkVySG91T0dPcVlsK3pCQ0lNWktQbHdJREFRQUJveU13SVRBT0JnTlYKSFE4QkFmOEVCQU1DQWdRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBTkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQQpkNUN5WUErbCtYVy9QckErZ0ViTWdNVnVFdnRVYzF2ay81U0RVaCs0MmZlY1pyeldjaVBvMUVKaGlRWE00NzMwCmtOTHZYYkhEYldDK2VSRFo0SUhWazV6RDl1YVBpbnU5RTRJNERvVWoydm1VOXcyQnA0dldXNnJTSVJGMmFSdW8KOGhyR2RyblNGOEdSaTZGTzFoV0xsVnJzK3RjbEhxMHdiU1hhVjNTNGhKNjd6cmE4N0IyajJ6eUZ0R0Z2dU1EdAp6VUpDOFlDTXRNMUcxNCsrWUxrVXNIbGhmMzJWanRZcmtoWHVxS2N4d05ObEcxeURsa05kdjYwRFNaZUUzNmlsCjRha2s1dC90Z200RllFMXc4b0RJNFZLM2d0cjcxNkIveWVIelIxbHJpOU1BZCtZWU1ZNUZuU0tNYStOd3EzYXYKbFNDVTBkeUdUK21WQXRXa0d0VjQrZz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K rules: - operations: ["CREATE", "UPDATE"] apiGroups: ["kubefledged.k8s.io"] diff --git a/hack/.golint_failures b/hack/.golint_failures index b0ec1385..a52232a5 100755 --- a/hack/.golint_failures +++ b/hack/.golint_failures @@ -1,2 +1 @@ -pkg/apis/fledged -pkg/apis/fledged/v1alpha1 +pkg/apis/kubefledged/v1alpha1 diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index ebb5422d..d41571fb 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -31,7 +31,7 @@ CODEGEN_PKG=${CODEGEN_PKG:-$(cd ${SCRIPT_ROOT}; ls -d -1 $GOPATH/pkg/mod/k8s.io/ chmod u+x ${CODEGEN_PKG}/generate-groups.sh ${CODEGEN_PKG}/generate-groups.sh "deepcopy,client,informer,lister" \ github.com/senthilrch/kube-fledged/pkg/client github.com/senthilrch/kube-fledged/pkg/apis \ - fledged:v1alpha1 \ + kubefledged:v1alpha1 \ --output-base "$(dirname ${BASH_SOURCE})/../../../.." \ --go-header-file ${SCRIPT_ROOT}/hack/boilerplate/boilerplate.generatego.txt diff --git a/pkg/apis/fledged/register.go b/pkg/apis/kubefledged/register.go similarity index 84% rename from pkg/apis/fledged/register.go rename to pkg/apis/kubefledged/register.go index 58ae4327..11695cb6 100644 --- a/pkg/apis/fledged/register.go +++ b/pkg/apis/kubefledged/register.go @@ -14,8 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ -package fledged +package kubefledged const ( - GroupName = "fledged.k8s.io" + //GroupName denotes the api group of the resource + GroupName = "kubefledged.k8s.io" ) diff --git a/pkg/apis/fledged/v1alpha1/doc.go b/pkg/apis/kubefledged/v1alpha1/doc.go similarity index 100% rename from pkg/apis/fledged/v1alpha1/doc.go rename to pkg/apis/kubefledged/v1alpha1/doc.go diff --git a/pkg/apis/fledged/v1alpha1/register.go b/pkg/apis/kubefledged/v1alpha1/register.go similarity index 90% rename from pkg/apis/fledged/v1alpha1/register.go rename to pkg/apis/kubefledged/v1alpha1/register.go index bb770bf3..ddb0de7a 100644 --- a/pkg/apis/fledged/v1alpha1/register.go +++ b/pkg/apis/kubefledged/v1alpha1/register.go @@ -21,11 +21,11 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - fledged "github.com/senthilrch/kube-fledged/pkg/apis/fledged" + kubefledged "github.com/senthilrch/kube-fledged/pkg/apis/kubefledged" ) // SchemeGroupVersion is group version used to register these objects -var SchemeGroupVersion = schema.GroupVersion{Group: fledged.GroupName, Version: "v1alpha1"} +var SchemeGroupVersion = schema.GroupVersion{Group: kubefledged.GroupName, Version: "v1alpha1"} // Kind takes an unqualified kind and returns back a Group qualified GroupKind func Kind(kind string) schema.GroupKind { diff --git a/pkg/apis/fledged/v1alpha1/types.go b/pkg/apis/kubefledged/v1alpha1/types.go similarity index 100% rename from pkg/apis/fledged/v1alpha1/types.go rename to pkg/apis/kubefledged/v1alpha1/types.go diff --git a/pkg/apis/fledged/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/kubefledged/v1alpha1/zz_generated.deepcopy.go similarity index 100% rename from pkg/apis/fledged/v1alpha1/zz_generated.deepcopy.go rename to pkg/apis/kubefledged/v1alpha1/zz_generated.deepcopy.go diff --git a/pkg/client/clientset/versioned/clientset.go b/pkg/client/clientset/versioned/clientset.go index 7fec2633..c1e33be3 100644 --- a/pkg/client/clientset/versioned/clientset.go +++ b/pkg/client/clientset/versioned/clientset.go @@ -21,7 +21,7 @@ package versioned import ( "fmt" - fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/typed/fledged/v1alpha1" + fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/typed/kubefledged/v1alpha1" discovery "k8s.io/client-go/discovery" rest "k8s.io/client-go/rest" flowcontrol "k8s.io/client-go/util/flowcontrol" diff --git a/pkg/client/clientset/versioned/fake/clientset_generated.go b/pkg/client/clientset/versioned/fake/clientset_generated.go index 7558650b..5f6e9b74 100644 --- a/pkg/client/clientset/versioned/fake/clientset_generated.go +++ b/pkg/client/clientset/versioned/fake/clientset_generated.go @@ -20,8 +20,8 @@ package fake import ( clientset "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned" - fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/typed/fledged/v1alpha1" - fakefledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/typed/fledged/v1alpha1/fake" + fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/typed/kubefledged/v1alpha1" + fakefledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/typed/kubefledged/v1alpha1/fake" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/discovery" diff --git a/pkg/client/clientset/versioned/fake/register.go b/pkg/client/clientset/versioned/fake/register.go index 66024529..650ae809 100644 --- a/pkg/client/clientset/versioned/fake/register.go +++ b/pkg/client/clientset/versioned/fake/register.go @@ -19,7 +19,7 @@ limitations under the License. package fake import ( - fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/kubefledged/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/client/clientset/versioned/scheme/register.go b/pkg/client/clientset/versioned/scheme/register.go index 07a92647..eb826c1f 100644 --- a/pkg/client/clientset/versioned/scheme/register.go +++ b/pkg/client/clientset/versioned/scheme/register.go @@ -19,7 +19,7 @@ limitations under the License. package scheme import ( - fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/kubefledged/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/client/clientset/versioned/typed/fledged/v1alpha1/doc.go b/pkg/client/clientset/versioned/typed/kubefledged/v1alpha1/doc.go similarity index 100% rename from pkg/client/clientset/versioned/typed/fledged/v1alpha1/doc.go rename to pkg/client/clientset/versioned/typed/kubefledged/v1alpha1/doc.go diff --git a/pkg/client/clientset/versioned/typed/fledged/v1alpha1/fake/doc.go b/pkg/client/clientset/versioned/typed/kubefledged/v1alpha1/fake/doc.go similarity index 100% rename from pkg/client/clientset/versioned/typed/fledged/v1alpha1/fake/doc.go rename to pkg/client/clientset/versioned/typed/kubefledged/v1alpha1/fake/doc.go diff --git a/pkg/client/clientset/versioned/typed/fledged/v1alpha1/fake/fake_imagecache.go b/pkg/client/clientset/versioned/typed/kubefledged/v1alpha1/fake/fake_imagecache.go similarity index 98% rename from pkg/client/clientset/versioned/typed/fledged/v1alpha1/fake/fake_imagecache.go rename to pkg/client/clientset/versioned/typed/kubefledged/v1alpha1/fake/fake_imagecache.go index 2b4bded2..77824738 100644 --- a/pkg/client/clientset/versioned/typed/fledged/v1alpha1/fake/fake_imagecache.go +++ b/pkg/client/clientset/versioned/typed/kubefledged/v1alpha1/fake/fake_imagecache.go @@ -19,7 +19,7 @@ limitations under the License. package fake import ( - v1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + v1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/kubefledged/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" schema "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/client/clientset/versioned/typed/fledged/v1alpha1/fake/fake_fledged_client.go b/pkg/client/clientset/versioned/typed/kubefledged/v1alpha1/fake/fake_kubefledged_client.go similarity index 96% rename from pkg/client/clientset/versioned/typed/fledged/v1alpha1/fake/fake_fledged_client.go rename to pkg/client/clientset/versioned/typed/kubefledged/v1alpha1/fake/fake_kubefledged_client.go index 83606f9d..f1a2779a 100644 --- a/pkg/client/clientset/versioned/typed/fledged/v1alpha1/fake/fake_fledged_client.go +++ b/pkg/client/clientset/versioned/typed/kubefledged/v1alpha1/fake/fake_kubefledged_client.go @@ -19,7 +19,7 @@ limitations under the License. package fake import ( - v1alpha1 "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/typed/fledged/v1alpha1" + v1alpha1 "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/typed/kubefledged/v1alpha1" rest "k8s.io/client-go/rest" testing "k8s.io/client-go/testing" ) diff --git a/pkg/client/clientset/versioned/typed/fledged/v1alpha1/generated_expansion.go b/pkg/client/clientset/versioned/typed/kubefledged/v1alpha1/generated_expansion.go similarity index 100% rename from pkg/client/clientset/versioned/typed/fledged/v1alpha1/generated_expansion.go rename to pkg/client/clientset/versioned/typed/kubefledged/v1alpha1/generated_expansion.go diff --git a/pkg/client/clientset/versioned/typed/fledged/v1alpha1/imagecache.go b/pkg/client/clientset/versioned/typed/kubefledged/v1alpha1/imagecache.go similarity index 98% rename from pkg/client/clientset/versioned/typed/fledged/v1alpha1/imagecache.go rename to pkg/client/clientset/versioned/typed/kubefledged/v1alpha1/imagecache.go index 0ea8e4fb..7a4c6f5e 100644 --- a/pkg/client/clientset/versioned/typed/fledged/v1alpha1/imagecache.go +++ b/pkg/client/clientset/versioned/typed/kubefledged/v1alpha1/imagecache.go @@ -21,7 +21,7 @@ package v1alpha1 import ( "time" - v1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + v1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/kubefledged/v1alpha1" scheme "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/scheme" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" diff --git a/pkg/client/clientset/versioned/typed/fledged/v1alpha1/fledged_client.go b/pkg/client/clientset/versioned/typed/kubefledged/v1alpha1/kubefledged_client.go similarity index 96% rename from pkg/client/clientset/versioned/typed/fledged/v1alpha1/fledged_client.go rename to pkg/client/clientset/versioned/typed/kubefledged/v1alpha1/kubefledged_client.go index 5f5fcc2e..253a9656 100644 --- a/pkg/client/clientset/versioned/typed/fledged/v1alpha1/fledged_client.go +++ b/pkg/client/clientset/versioned/typed/kubefledged/v1alpha1/kubefledged_client.go @@ -19,7 +19,7 @@ limitations under the License. package v1alpha1 import ( - v1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + v1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/kubefledged/v1alpha1" "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned/scheme" rest "k8s.io/client-go/rest" ) diff --git a/pkg/client/informers/externalversions/factory.go b/pkg/client/informers/externalversions/factory.go index 019c26c3..c8bc16b1 100644 --- a/pkg/client/informers/externalversions/factory.go +++ b/pkg/client/informers/externalversions/factory.go @@ -24,8 +24,8 @@ import ( time "time" versioned "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned" - fledged "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions/fledged" internalinterfaces "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions/internalinterfaces" + kubefledged "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions/kubefledged" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -172,9 +172,9 @@ type SharedInformerFactory interface { ForResource(resource schema.GroupVersionResource) (GenericInformer, error) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool - Fledged() fledged.Interface + Fledged() kubefledged.Interface } -func (f *sharedInformerFactory) Fledged() fledged.Interface { - return fledged.New(f, f.namespace, f.tweakListOptions) +func (f *sharedInformerFactory) Fledged() kubefledged.Interface { + return kubefledged.New(f, f.namespace, f.tweakListOptions) } diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go index ff9f375d..4e5fccd0 100644 --- a/pkg/client/informers/externalversions/generic.go +++ b/pkg/client/informers/externalversions/generic.go @@ -21,7 +21,7 @@ package externalversions import ( "fmt" - v1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + v1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/kubefledged/v1alpha1" schema "k8s.io/apimachinery/pkg/runtime/schema" cache "k8s.io/client-go/tools/cache" ) diff --git a/pkg/client/informers/externalversions/fledged/interface.go b/pkg/client/informers/externalversions/kubefledged/interface.go similarity index 96% rename from pkg/client/informers/externalversions/fledged/interface.go rename to pkg/client/informers/externalversions/kubefledged/interface.go index 87f08f54..c0d1dac4 100644 --- a/pkg/client/informers/externalversions/fledged/interface.go +++ b/pkg/client/informers/externalversions/kubefledged/interface.go @@ -16,11 +16,11 @@ limitations under the License. // Code generated by informer-gen. DO NOT EDIT. -package fledged +package kubefledged import ( - v1alpha1 "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions/fledged/v1alpha1" internalinterfaces "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions/internalinterfaces" + v1alpha1 "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions/kubefledged/v1alpha1" ) // Interface provides access to each of this group's versions. diff --git a/pkg/client/informers/externalversions/fledged/v1alpha1/imagecache.go b/pkg/client/informers/externalversions/kubefledged/v1alpha1/imagecache.go similarity index 93% rename from pkg/client/informers/externalversions/fledged/v1alpha1/imagecache.go rename to pkg/client/informers/externalversions/kubefledged/v1alpha1/imagecache.go index c8cccb05..0a370198 100644 --- a/pkg/client/informers/externalversions/fledged/v1alpha1/imagecache.go +++ b/pkg/client/informers/externalversions/kubefledged/v1alpha1/imagecache.go @@ -21,10 +21,10 @@ package v1alpha1 import ( time "time" - fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + kubefledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/kubefledged/v1alpha1" versioned "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned" internalinterfaces "github.com/senthilrch/kube-fledged/pkg/client/informers/externalversions/internalinterfaces" - v1alpha1 "github.com/senthilrch/kube-fledged/pkg/client/listers/fledged/v1alpha1" + v1alpha1 "github.com/senthilrch/kube-fledged/pkg/client/listers/kubefledged/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" watch "k8s.io/apimachinery/pkg/watch" @@ -70,7 +70,7 @@ func NewFilteredImageCacheInformer(client versioned.Interface, namespace string, return client.FledgedV1alpha1().ImageCaches(namespace).Watch(options) }, }, - &fledgedv1alpha1.ImageCache{}, + &kubefledgedv1alpha1.ImageCache{}, resyncPeriod, indexers, ) @@ -81,7 +81,7 @@ func (f *imageCacheInformer) defaultInformer(client versioned.Interface, resyncP } func (f *imageCacheInformer) Informer() cache.SharedIndexInformer { - return f.factory.InformerFor(&fledgedv1alpha1.ImageCache{}, f.defaultInformer) + return f.factory.InformerFor(&kubefledgedv1alpha1.ImageCache{}, f.defaultInformer) } func (f *imageCacheInformer) Lister() v1alpha1.ImageCacheLister { diff --git a/pkg/client/informers/externalversions/fledged/v1alpha1/interface.go b/pkg/client/informers/externalversions/kubefledged/v1alpha1/interface.go similarity index 100% rename from pkg/client/informers/externalversions/fledged/v1alpha1/interface.go rename to pkg/client/informers/externalversions/kubefledged/v1alpha1/interface.go diff --git a/pkg/client/listers/fledged/v1alpha1/expansion_generated.go b/pkg/client/listers/kubefledged/v1alpha1/expansion_generated.go similarity index 100% rename from pkg/client/listers/fledged/v1alpha1/expansion_generated.go rename to pkg/client/listers/kubefledged/v1alpha1/expansion_generated.go diff --git a/pkg/client/listers/fledged/v1alpha1/imagecache.go b/pkg/client/listers/kubefledged/v1alpha1/imagecache.go similarity index 97% rename from pkg/client/listers/fledged/v1alpha1/imagecache.go rename to pkg/client/listers/kubefledged/v1alpha1/imagecache.go index e6f1423e..f6aaae15 100644 --- a/pkg/client/listers/fledged/v1alpha1/imagecache.go +++ b/pkg/client/listers/kubefledged/v1alpha1/imagecache.go @@ -19,7 +19,7 @@ limitations under the License. package v1alpha1 import ( - v1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + v1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/kubefledged/v1alpha1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" diff --git a/pkg/images/image_helpers.go b/pkg/images/image_helpers.go index a0aa3ef8..cd58785b 100644 --- a/pkg/images/image_helpers.go +++ b/pkg/images/image_helpers.go @@ -23,7 +23,7 @@ import ( "time" "github.com/golang/glog" - fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/kubefledged/v1alpha1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/images/image_manager.go b/pkg/images/image_manager.go index 0ab6161b..aa54a201 100644 --- a/pkg/images/image_manager.go +++ b/pkg/images/image_manager.go @@ -23,7 +23,7 @@ import ( "time" "github.com/golang/glog" - fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/kubefledged/v1alpha1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/images/image_manager_test.go b/pkg/images/image_manager_test.go index f589512c..766084f6 100644 --- a/pkg/images/image_manager_test.go +++ b/pkg/images/image_manager_test.go @@ -23,7 +23,7 @@ import ( "testing" "time" - fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/kubefledged/v1alpha1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/webhook/imagecache.go b/pkg/webhook/imagecache.go index e0000783..96035049 100644 --- a/pkg/webhook/imagecache.go +++ b/pkg/webhook/imagecache.go @@ -22,7 +22,7 @@ import ( "reflect" "github.com/golang/glog" - fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/fledged/v1alpha1" + fledgedv1alpha1 "github.com/senthilrch/kube-fledged/pkg/apis/kubefledged/v1alpha1" v1 "k8s.io/api/admission/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) From 6209f2e40fdc09cf10dd54144a3c37682fa0e2d7 Mon Sep 17 00:00:00 2001 From: bhuvanessr Date: Fri, 29 May 2020 17:59:17 +0530 Subject: [PATCH 17/21] update helm chart --- Makefile | 109 ++++++++++-------- .../deploy/clusterrole.yaml | 17 ++- .../deploy/clusterrole_binding.yaml | 2 +- ...s.helm.k8s.io_v1alpha1_kubefledged_cr.yaml | 7 +- .../kubefledged-operator/deploy/operator.yaml | 2 +- .../deploy/service_account.yaml | 2 +- .../kubefledged/templates/_helpers.tpl | 29 +++++ .../kubefledged/templates/clusterrole.yaml | 4 +- .../kubefledged/templates/crd.yaml | 4 +- ...oyment.yaml => deployment-controller.yaml} | 22 ++-- .../templates/deployment-webhook-server.yaml | 63 ++++++++++ .../templates/service-webhook-server.yaml | 18 +++ .../templates/validatingwebhook.yaml | 25 ++++ .../helm-charts/kubefledged/values.yaml | 42 +++++-- deploy/kubefledged-validatingwebhook.yaml | 2 +- deploy/webhook-patch-ca-bundle.sh | 6 +- 16 files changed, 269 insertions(+), 85 deletions(-) rename deploy/kubefledged-operator/helm-charts/kubefledged/templates/{deployment.yaml => deployment-controller.yaml} (62%) create mode 100644 deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-webhook-server.yaml create mode 100644 deploy/kubefledged-operator/helm-charts/kubefledged/templates/service-webhook-server.yaml create mode 100644 deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml diff --git a/Makefile b/Makefile index cac098cf..b870c5a9 100644 --- a/Makefile +++ b/Makefile @@ -63,7 +63,7 @@ ifndef ALPINE_VERSION endif ifndef OPERATORSDK_VERSION - OPERATORSDK_VERSION=v0.17.0 + OPERATORSDK_VERSION=v0.17.1 endif ifndef TARGET_PLATFORMS @@ -78,6 +78,14 @@ ifndef BUILD_OUTPUT BUILD_OUTPUT=--push endif +ifndef OPERATOR_NAMESPACE + OPERATOR_NAMESPACE=kubefledged-operator +endif + +ifndef KUBEFLEDGED_NAMESPACE + KUBEFLEDGED_NAMESPACE=kube-fledged +endif + HTTP_PROXY_CONFIG= ifdef HTTP_PROXY HTTP_PROXY_CONFIG=--build-arg http_proxy=${HTTP_PROXY} @@ -164,62 +172,65 @@ test: deploy-using-yaml: -kubectl apply -f deploy/kubefledged-namespace.yaml - bash deploy/webhook-create-signed-cert.sh --namespace kube-fledged --service kubefledged-webhook-server --secret kubefledged-webhook-server && \ - bash deploy/webhook-patch-ca-bundle.sh && \ - kubectl apply -f deploy/kubefledged-crd.yaml && \ - kubectl apply -f deploy/kubefledged-serviceaccount.yaml && \ - kubectl apply -f deploy/kubefledged-clusterrole.yaml && \ - kubectl apply -f deploy/kubefledged-clusterrolebinding.yaml && \ - kubectl apply -f deploy/kubefledged-deployment-controller.yaml && \ - kubectl apply -f deploy/kubefledged-deployment-webhook-server.yaml && \ - kubectl apply -f deploy/kubefledged-service-webhook-server.yaml && \ + bash deploy/webhook-create-signed-cert.sh --namespace kube-fledged --service kubefledged-webhook-server --secret kubefledged-webhook-server + bash deploy/webhook-patch-ca-bundle.sh + kubectl apply -f deploy/kubefledged-crd.yaml + kubectl apply -f deploy/kubefledged-serviceaccount.yaml + kubectl apply -f deploy/kubefledged-clusterrole.yaml + kubectl apply -f deploy/kubefledged-clusterrolebinding.yaml + kubectl apply -f deploy/kubefledged-deployment-controller.yaml + kubectl apply -f deploy/kubefledged-deployment-webhook-server.yaml + kubectl apply -f deploy/kubefledged-service-webhook-server.yaml kubectl apply -f deploy/kubefledged-validatingwebhook.yaml deploy-using-operator: - # Deploy the operator to a separate namespace called "operators" - sed -i "s|OPERATOR_NAMESPACE|operators|g" deploy/kubefledged-operator/deploy/service_account.yaml - sed -i "s|OPERATOR_NAMESPACE|operators|g" deploy/kubefledged-operator/deploy/clusterrole_binding.yaml - sed -i "s|OPERATOR_NAMESPACE|operators|g" deploy/kubefledged-operator/deploy/operator.yaml - -kubectl create namespace operators - kubectl create -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml - kubectl create -f deploy/kubefledged-operator/deploy/service_account.yaml - kubectl create -f deploy/kubefledged-operator/deploy/clusterrole.yaml - kubectl create -f deploy/kubefledged-operator/deploy/clusterrole_binding.yaml - kubectl create -f deploy/kubefledged-operator/deploy/operator.yaml - # Deploy kube-fledged to a separate namespace called "kube-fledged" - sed -i "s|OPERATOR_NAMESPACE|operators|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml - sed -i "s|KUBEFLEDGED_NAMESPACE|kube-fledged|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml - -kubectl create namespace kube-fledged - kubectl create -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml + # Create the namespaces for operator and kubefledged + -kubectl create namespace ${OPERATOR_NAMESPACE} + -kubectl create namespace ${KUBEFLEDGED_NAMESPACE} + # Deploy the operator to a separate namespace + sed -i 's|{{OPERATOR_NAMESPACE}}|${OPERATOR_NAMESPACE}|g' deploy/kubefledged-operator/deploy/service_account.yaml + sed -i "s|{{OPERATOR_NAMESPACE}}|${OPERATOR_NAMESPACE}|g" deploy/kubefledged-operator/deploy/clusterrole_binding.yaml + sed -i "s|{{OPERATOR_NAMESPACE}}|${OPERATOR_NAMESPACE}|g" deploy/kubefledged-operator/deploy/operator.yaml + kubectl apply -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml + kubectl apply -f deploy/kubefledged-operator/deploy/service_account.yaml + kubectl apply -f deploy/kubefledged-operator/deploy/clusterrole.yaml + kubectl apply -f deploy/kubefledged-operator/deploy/clusterrole_binding.yaml + kubectl apply -f deploy/kubefledged-operator/deploy/operator.yaml + # Deploy kube-fledged to a separate namespace + sed -i "s|{{OPERATOR_NAMESPACE}}|${OPERATOR_NAMESPACE}|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml + sed -i "s|{{KUBEFLEDGED_NAMESPACE}}|${KUBEFLEDGED_NAMESPACE}|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml + bash deploy/webhook-create-signed-cert.sh --namespace ${KUBEFLEDGED_NAMESPACE} --service kubefledged-webhook-server --secret kubefledged-webhook-server + bash deploy/webhook-patch-ca-bundle.sh + kubectl apply -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml update: - kubectl scale deployment kubefledged-controller --replicas=0 -n kube-fledged && \ - kubectl scale deployment kubefledged-webhook-server --replicas=0 -n kube-fledged && sleep 1 && \ - kubectl scale deployment kubefledged-controller --replicas=1 -n kube-fledged && sleep 1 && \ - kubectl scale deployment kubefledged-webhook-server --replicas=1 -n kube-fledged && sleep 1 && \ + kubectl scale deployment kubefledged-controller --replicas=0 -n kube-fledged + kubectl scale deployment kubefledged-webhook-server --replicas=0 -n kube-fledged && sleep 1 + kubectl scale deployment kubefledged-controller --replicas=1 -n kube-fledged && sleep 1 + kubectl scale deployment kubefledged-webhook-server --replicas=1 -n kube-fledged && sleep 1 kubectl get pods -l app=kubefledged -n kube-fledged remove: - kubectl delete -f deploy/kubefledged-namespace.yaml && \ - kubectl delete -f deploy/kubefledged-clusterrolebinding.yaml && \ - kubectl delete -f deploy/kubefledged-clusterrole.yaml && \ - kubectl delete -f deploy/kubefledged-crd.yaml && \ - kubectl delete -f deploy/kubefledged-validatingwebhook.yaml + -kubectl delete -f deploy/kubefledged-namespace.yaml + -kubectl delete -f deploy/kubefledged-clusterrolebinding.yaml + -kubectl delete -f deploy/kubefledged-clusterrole.yaml + -kubectl delete -f deploy/kubefledged-crd.yaml + -kubectl delete -f deploy/kubefledged-validatingwebhook.yaml remove-all: - # Remove kube-fledged and the namespace "kube-fledged" - kubectl delete -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml - -kubectl delete namespace kube-fledged - sed -i "s|kube-fledged|KUBEFLEDGED_NAMESPACE|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml - sed -i "s|operators|OPERATOR_NAMESPACE|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml - # Remove the operator and the namespace "operators" - kubectl delete -f deploy/kubefledged-operator/deploy/operator.yaml - kubectl delete -f deploy/kubefledged-operator/deploy/clusterrole_binding.yaml - kubectl delete -f deploy/kubefledged-operator/deploy/clusterrole.yaml - kubectl delete -f deploy/kubefledged-operator/deploy/service_account.yaml - kubectl delete -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml - -kubectl delete namespace operators - sed -i "s|operators|OPERATOR_NAMESPACE|g" deploy/kubefledged-operator/deploy/operator.yaml - sed -i "s|operators|OPERATOR_NAMESPACE|g" deploy/kubefledged-operator/deploy/clusterrole_binding.yaml - sed -i "s|operators|OPERATOR_NAMESPACE|g" deploy/kubefledged-operator/deploy/service_account.yaml + # Remove kubefledged and the namespace + -kubectl delete -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml + -kubectl delete namespace ${KUBEFLEDGED_NAMESPACE} + -sed -i "s|${KUBEFLEDGED_NAMESPACE}|{{KUBEFLEDGED_NAMESPACE}}|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml + -sed -i "s|${OPERATOR_NAMESPACE}|{{OPERATOR_NAMESPACE}}|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml + # Remove the kubefledged-operator and the namespace + -kubectl delete -f deploy/kubefledged-operator/deploy/operator.yaml + -kubectl delete -f deploy/kubefledged-operator/deploy/clusterrole_binding.yaml + -kubectl delete -f deploy/kubefledged-operator/deploy/clusterrole.yaml + -kubectl delete -f deploy/kubefledged-operator/deploy/service_account.yaml + -kubectl delete -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml + -kubectl delete namespace ${OPERATOR_NAMESPACE} + -sed -i "s|${OPERATOR_NAMESPACE}|{{OPERATOR_NAMESPACE}}|g" deploy/kubefledged-operator/deploy/operator.yaml + -sed -i "s|${OPERATOR_NAMESPACE}|{{OPERATOR_NAMESPACE}}|g" deploy/kubefledged-operator/deploy/clusterrole_binding.yaml + -sed -i "s|${OPERATOR_NAMESPACE}|{{OPERATOR_NAMESPACE}}|g" deploy/kubefledged-operator/deploy/service_account.yaml diff --git a/deploy/kubefledged-operator/deploy/clusterrole.yaml b/deploy/kubefledged-operator/deploy/clusterrole.yaml index 9b82197e..18343947 100644 --- a/deploy/kubefledged-operator/deploy/clusterrole.yaml +++ b/deploy/kubefledged-operator/deploy/clusterrole.yaml @@ -1,7 +1,6 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - creationTimestamp: null name: kubefledged-operator rules: - apiGroups: @@ -104,7 +103,7 @@ rules: - get - patch - apiGroups: - - "fledged.k8s.io" + - "kubefledged.k8s.io" resources: - imagecaches verbs: @@ -114,7 +113,7 @@ rules: - update - patch - apiGroups: - - "fledged.k8s.io" + - "kubefledged.k8s.io" resources: - imagecaches/status verbs: @@ -146,4 +145,14 @@ rules: - list - create - delete - +- apiGroups: + - "admissionregistration.k8s.io" + resources: + - validatingwebhookconfigurations + verbs: + - get + - list + - create + - update + - patch + - delete diff --git a/deploy/kubefledged-operator/deploy/clusterrole_binding.yaml b/deploy/kubefledged-operator/deploy/clusterrole_binding.yaml index 7a2d2c8c..8ca12f66 100644 --- a/deploy/kubefledged-operator/deploy/clusterrole_binding.yaml +++ b/deploy/kubefledged-operator/deploy/clusterrole_binding.yaml @@ -5,7 +5,7 @@ metadata: subjects: - kind: ServiceAccount name: kubefledged-operator - namespace: OPERATOR_NAMESPACE + namespace: {{OPERATOR_NAMESPACE}} roleRef: kind: ClusterRole name: kubefledged-operator diff --git a/deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml b/deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml index 121d0bb7..4f31dca5 100644 --- a/deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml +++ b/deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml @@ -1,8 +1,9 @@ apiVersion: charts.helm.k8s.io/v1alpha1 kind: KubeFledged metadata: - name: mykubefledged - namespace: OPERATOR_NAMESPACE + name: kubefledged + namespace: {{OPERATOR_NAMESPACE}} spec: # Defaults defined in /helm-charts/kubefledged/values.yaml - kubefledgedNameSpace: KUBEFLEDGED_NAMESPACE + kubefledgedNameSpace: {{KUBEFLEDGED_NAMESPACE}} + validatingWebhookCABundle: {{CA_BUNDLE}} diff --git a/deploy/kubefledged-operator/deploy/operator.yaml b/deploy/kubefledged-operator/deploy/operator.yaml index 92a48cb3..2b3962d7 100644 --- a/deploy/kubefledged-operator/deploy/operator.yaml +++ b/deploy/kubefledged-operator/deploy/operator.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: kubefledged-operator - namespace: OPERATOR_NAMESPACE + namespace: {{OPERATOR_NAMESPACE}} spec: replicas: 1 selector: diff --git a/deploy/kubefledged-operator/deploy/service_account.yaml b/deploy/kubefledged-operator/deploy/service_account.yaml index 1cb416f5..f35e251a 100644 --- a/deploy/kubefledged-operator/deploy/service_account.yaml +++ b/deploy/kubefledged-operator/deploy/service_account.yaml @@ -2,4 +2,4 @@ apiVersion: v1 kind: ServiceAccount metadata: name: kubefledged-operator - namespace: OPERATOR_NAMESPACE + namespace: {{OPERATOR_NAMESPACE}} diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/_helpers.tpl b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/_helpers.tpl index 824d7fa9..ce5116f2 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/_helpers.tpl +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/_helpers.tpl @@ -83,3 +83,32 @@ Create the name of the cluster role binding to use {{ default "default" .Values.clusterRoleBinding.name }} {{- end -}} {{- end -}} + +{{/* +Create the name of the validating webhook configuration to use +*/}} +{{- define "kubefledged.validatingWebhookName" -}} +{{- if .Values.validatingWebhook.create -}} + {{ default (include "kubefledged.fullname" .) .Values.validatingWebhook.name }} +{{- else -}} + {{ default "default" .Values.validatingWebhook.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service for the webhook server to use +*/}} +{{- define "kubefledged.webhookServiceName" -}} +{{- if .Values.webhookService.create -}} + {{ default (include "kubefledged.fullname" .) .Values.webhookService.name }} +{{- else -}} + {{ default "default" .Values.webhookService.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the secret containing the webhook server's keypair +*/}} +{{- define "kubefledged.secretName" -}} +{{ default (include "kubefledged.fullname" .) .Values.secret.name }} +{{- end -}} diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/clusterrole.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/clusterrole.yaml index d09b5c45..458d63ba 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/clusterrole.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/clusterrole.yaml @@ -9,7 +9,7 @@ metadata: rbac.authorization.kubernetes.io/autoupdate: "true" rules: - apiGroups: - - "fledged.k8s.io" + - "kubefledged.k8s.io" resources: - imagecaches verbs: @@ -18,7 +18,7 @@ rules: - watch - update - apiGroups: - - "fledged.k8s.io" + - "kubefledged.k8s.io" resources: - imagecaches/status verbs: diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/crd.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/crd.yaml index 89198a4b..ccecf759 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/crd.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/crd.yaml @@ -1,11 +1,11 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: - name: imagecaches.fledged.k8s.io + name: imagecaches.kubefledged.k8s.io labels: {{ include "kubefledged.labels" . | nindent 4 }} spec: - group: fledged.k8s.io + group: kubefledged.k8s.io versions: - name: v1alpha1 served: true diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-controller.yaml similarity index 62% rename from deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment.yaml rename to deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-controller.yaml index b2692105..b818ecbe 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-controller.yaml @@ -1,19 +1,19 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "kubefledged.fullname" . }} + name: {{ include "kubefledged.fullname" . }}-controller labels: {{- include "kubefledged.labels" . | nindent 4 }} namespace: {{ .Values.kubefledgedNameSpace }} spec: - replicas: {{ .Values.replicaCount }} + replicas: {{ .Values.controllerReplicaCount }} selector: matchLabels: - {{- include "kubefledged.selectorLabels" . | nindent 6 }} + {{- include "kubefledged.selectorLabels" . | nindent 6 }}-controller template: metadata: labels: - {{- include "kubefledged.selectorLabels" . | nindent 8 }} + {{- include "kubefledged.selectorLabels" . | nindent 8 }}-controller spec: {{- with .Values.imagePullSecrets }} imagePullSecrets: @@ -26,14 +26,14 @@ spec: - name: {{ .Chart.Name }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} - image: {{ .Values.image.fledgedRepository }}:{{ .Chart.AppVersion }} - command: {{ .Values.command }} + image: {{ .Values.image.kubefledgedControllerRepository }}:{{ .Chart.AppVersion }} + command: {{ .Values.command.kubefledgedControllerCommand }} args: - - "--stderrthreshold={{ .Values.args.logLevel}}" - - "--image-pull-deadline-duration={{ .Values.args.imagePullDeadlineDuration}}" - - "--image-cache-refresh-frequency={{ .Values.args.imageCacheRefreshFrequency}}" - - "--docker-client-image={{ .Values.image.dockerClientRepository }}:{{ .Chart.AppVersion }}" - - "--image-pull-policy={{ .Values.args.imagePullPolicy}}" + - "--stderrthreshold={{ .Values.args.controllerLogLevel }}" + - "--image-pull-deadline-duration={{ .Values.args.controllerImagePullDeadlineDuration }}" + - "--image-cache-refresh-frequency={{ .Values.args.controllerImageCacheRefreshFrequency }}" + - "--cri-client-image={{ .Values.image.kubefledgedCRIClientRepository }}:{{ .Chart.AppVersion }}" + - "--image-pull-policy={{ .Values.args.controllerImagePullPolicy }}" imagePullPolicy: {{ .Values.image.pullPolicy }} env: - name: KUBEFLEDGED_NAMESPACE diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-webhook-server.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-webhook-server.yaml new file mode 100644 index 00000000..7a34102a --- /dev/null +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/deployment-webhook-server.yaml @@ -0,0 +1,63 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "kubefledged.fullname" . }}-webhook-server + labels: + {{- include "kubefledged.labels" . | nindent 4 }} + namespace: {{ .Values.kubefledgedNameSpace }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "kubefledged.selectorLabels" . | nindent 6 }}-webhook-server + template: + metadata: + labels: + {{- include "kubefledged.selectorLabels" . | nindent 8 }}-webhook-server + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "kubefledged.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: {{ .Values.image.kubefledgedWebhookServerRepository }}:{{ .Chart.AppVersion }} + command: {{ .Values.command.kubefledgedWebhookServerCommand }} + args: + - "--stderrthreshold={{ .Values.args.webhookServerLogLevel }}" + - "--cert-file={{ .Values.args.webhookServerCertFile }}" + - "--key-file={{ .Values.args.webhookServerKeyFile }}" + - "--port={{ .Values.args.webhookServerPort }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: KUBEFLEDGED_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: secret-volume + mountPath: "/var/run/secrets/webhook-server" + readOnly: true + volumes: + - name: secret-volume + secret: + secretName: kubefledged-webhook-server + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/service-webhook-server.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/service-webhook-server.yaml new file mode 100644 index 00000000..9bc4dc56 --- /dev/null +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/service-webhook-server.yaml @@ -0,0 +1,18 @@ +{{- if .Values.webhookService.create -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "kubefledged.webhookServiceName" . }}-webhook-server + labels: + {{ include "kubefledged.labels" . | nindent 4 }} + namespace: {{ .Values.kubefledgedNameSpace }} +spec: + ports: + - name: webhook-server + port: {{ .Values.webhookService.port }} + protocol: TCP + targetPort: {{ .Values.webhookService.targetPort }} + selector: + {{- include "kubefledged.selectorLabels" . | nindent 4 }}-webhook-server + type: ClusterIP +{{- end -}} diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml new file mode 100644 index 00000000..b5105b86 --- /dev/null +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml @@ -0,0 +1,25 @@ +{{- if .Values.validatingWebhook.create -}} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: {{ include "kubefledged.validatingWebhookName" . }} +webhooks: + - name: validate-image-cache.kubefledged.k8s.io + admissionReviewVersions: ["v1"] + timeoutSeconds: 1 + failurePolicy: Fail + sideEffects: None + clientConfig: + service: + namespace: kube-fledged + name: {{ include "kubefledged.webhookServiceName" . }}-webhook-server + path: "/validate-image-cache" + port: {{ .Values.webhookService.port }} + caBundle: {{ .Values.validatingWebhookCABundle }} + rules: + - operations: ["CREATE", "UPDATE"] + apiGroups: ["kubefledged.k8s.io"] + apiVersions: ["v1alpha1"] + resources: ["imagecaches"] + scope: "Namespaced" +{{- end -}} \ No newline at end of file diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/values.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/values.yaml index f0e42615..98346ce5 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/values.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/values.yaml @@ -2,18 +2,27 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. -replicaCount: 1 +controllerReplicaCount: 1 +webhookServerReplicaCount: 1 kubefledgedNameSpace: kube-fledged image: - fledgedRepository: docker.io/senthilrch/fledged - dockerClientRepository: docker.io/senthilrch/fledged-docker-client + kubefledgedControllerRepository: docker.io/senthilrch/kubefledged-controller + kubefledgedCRIClientRepository: docker.io/senthilrch/kubefledged-cri-client + kubefledgedWebhookServerRepository: docker.io/senthilrch/kubefledged-webhook-server pullPolicy: Always -command: ["/opt/bin/fledged"] +command: + kubefledgedControllerCommand: ["/opt/bin/kubefledged-controller"] + kubefledgedWebhookServerCommand: ["/opt/bin/kubefledged-webhook-server"] args: - logLevel: INFO - imagePullDeadlineDuration: 5m - imageCacheRefreshFrequency: 15m - imagePullPolicy: IfNotPresent + controllerLogLevel: INFO + controllerImagePullDeadlineDuration: 5m + controllerImageCacheRefreshFrequency: 15m + controllerImagePullPolicy: IfNotPresent + webhookServerLogLevel: INFO + webhookServerCertFile: /var/run/secrets/webhook-server/cert.pem + webhookServerKeyFile: /var/run/secrets/webhook-server/key.pem + webhookServerPort: 443 +validatingWebhookCABundle: imagePullSecrets: [] nameOverride: "" fullnameOverride: "" @@ -39,6 +48,16 @@ clusterRoleBinding: # If not set and create is true, a name is generated using the fullname template name: +validatingWebhook: + # Specifies whether a validating webhook configuration should be created + create: true + # The name of the validating webhook configuration to use. + # If not set and create is true, a name is generated using the fullname template + name: + +secret: + name: + podSecurityContext: {} # fsGroup: 2000 @@ -54,6 +73,13 @@ service: type: ClusterIP port: 80 +webhookService: + create: true + type: ClusterIP + port: 3443 + targetPort: 443 + name: + ingress: enabled: false annotations: {} diff --git a/deploy/kubefledged-validatingwebhook.yaml b/deploy/kubefledged-validatingwebhook.yaml index d0602abc..22e08ea9 100644 --- a/deploy/kubefledged-validatingwebhook.yaml +++ b/deploy/kubefledged-validatingwebhook.yaml @@ -14,7 +14,7 @@ webhooks: name: kubefledged-webhook-server path: "/validate-image-cache" port: 3443 - caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURERENDQWZTZ0F3SUJBZ0lSQU52ZlM5L0tWK3hJeEZ3blZneVNzVkl3RFFZSktvWklodmNOQVFFTEJRQXcKTHpFdE1Dc0dBMVVFQXhNa05HRXhaREUyTmprdE9XUmpaQzAwWmpoa0xXRTJPR1V0WVRoaFlXUXhNVGRtT1dSagpNQjRYRFRJd01EVXlPVEEwTlRNeE9Wb1hEVEkxTURVeU9EQTFOVE14T1Zvd0x6RXRNQ3NHQTFVRUF4TWtOR0V4ClpERTJOamt0T1dSalpDMDBaamhrTFdFMk9HVXRZVGhoWVdReE1UZG1PV1JqTUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDRNakpWQUFYakdwVkxnWmt2cEVCOEFubEdMaERHakNHYXQ5eHk0VApoc0JlbGVTeldkUlZHZVBGTW9HNERMMndGQmRoNThHUmlxNGtNMy9LdnVMNld3c3dRYzRYWHJXYXYwdjAzYVdIClFvbWRTUVFlSnJncXlYL3dxanFYVkZ2bGtoejR1UjRaaHNoekxiMEFpLzg2Vnc5cUI2cXZkNW9tbDZBSUJZVEkKTEVQcnJTbjlMb0NEb3JNMDQ5a3AxbldDMXY0OHlmSlJ5RUN4aTRGOVRZRnE1ZDcyU09tMGNtNVZDN3A5bllBVQpvTG5VWkRkc1BLdWFHNXZHaU5GSDk1VTM2aXNsekRTV09YaFZORi9TcUxOeGJDWGZsS0wwRzZLdi9YTllGc2NPCjhaR1ArZWdBSHNjMG1VMkhCc2NDcDcxRkVySG91T0dPcVlsK3pCQ0lNWktQbHdJREFRQUJveU13SVRBT0JnTlYKSFE4QkFmOEVCQU1DQWdRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBTkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQQpkNUN5WUErbCtYVy9QckErZ0ViTWdNVnVFdnRVYzF2ay81U0RVaCs0MmZlY1pyeldjaVBvMUVKaGlRWE00NzMwCmtOTHZYYkhEYldDK2VSRFo0SUhWazV6RDl1YVBpbnU5RTRJNERvVWoydm1VOXcyQnA0dldXNnJTSVJGMmFSdW8KOGhyR2RyblNGOEdSaTZGTzFoV0xsVnJzK3RjbEhxMHdiU1hhVjNTNGhKNjd6cmE4N0IyajJ6eUZ0R0Z2dU1EdAp6VUpDOFlDTXRNMUcxNCsrWUxrVXNIbGhmMzJWanRZcmtoWHVxS2N4d05ObEcxeURsa05kdjYwRFNaZUUzNmlsCjRha2s1dC90Z200RllFMXc4b0RJNFZLM2d0cjcxNkIveWVIelIxbHJpOU1BZCtZWU1ZNUZuU0tNYStOd3EzYXYKbFNDVTBkeUdUK21WQXRXa0d0VjQrZz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + caBundle: {{CA_BUNDLE}} rules: - operations: ["CREATE", "UPDATE"] apiGroups: ["kubefledged.k8s.io"] diff --git a/deploy/webhook-patch-ca-bundle.sh b/deploy/webhook-patch-ca-bundle.sh index 2df0b76c..3865c002 100644 --- a/deploy/webhook-patch-ca-bundle.sh +++ b/deploy/webhook-patch-ca-bundle.sh @@ -19,5 +19,7 @@ set -o nounset set -o pipefail export CA_BUNDLE=$(kubectl config view --raw --flatten -o json | jq -r '.clusters[] | select(.name == "'$(kubectl config current-context)'") | .cluster."certificate-authority-data"') - -sed -i "s|\${CA_BUNDLE}|${CA_BUNDLE}|g" deploy/kubefledged-validatingwebhook.yaml +#CA_DECODED=$(echo ${CA_BUNDLE} | base64 -d -) +sed -i "s|{{CA_BUNDLE}}|${CA_BUNDLE}|g" deploy/kubefledged-validatingwebhook.yaml +#sed -i "s|{{CA_BUNDLE}}|${CA_DECODED}|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml +sed -i "s|{{CA_BUNDLE}}|${CA_BUNDLE}|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml From 73d4ed9606544bfeb6e622133d2c4561447752c9 Mon Sep 17 00:00:00 2001 From: bhuvanessr Date: Sun, 31 May 2020 23:40:58 +0530 Subject: [PATCH 18/21] Update README --- README.md | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index f5921e6e..1b117561 100644 --- a/README.md +++ b/README.md @@ -52,9 +52,9 @@ _kube-fledged_ provides CRUD APIs to manage the lifecycle of the image cache, an ## Prerequisites -- A functioning kubernetes cluster (v1.9 or above). It could be a simple development cluster like minikube or a large production cluster. +- A functioning kubernetes cluster (v1.16 or above). It could be a simple development cluster like minikube or a large production cluster. - All master and worker nodes having the ["kubernetes.io/hostname"](https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/#kubernetes-io-hostname) label. -- git, make, go, docker engine (>= 19.03) and kubectl installed on a local linux machine. kubectl configured properly to access the cluster. +- git, make, go, docker engine (>= 19.03) and kubectl installed on a local linux machine. kubectl configured properly to access the cluster with cluster-admin privileges. ## Quick Install using YAML manifests @@ -94,7 +94,7 @@ These instructions install _kube-fledged_ to a separate namespace called "kube-f $ cd $HOME/src/github.com/senthilrch/kube-fledged ``` -- Deploy the operator to a separate namespace called "operators" and _kube-fledged_ to a separate namespace called "kube-fledged" +- Deploy the helm operator to a separate namespace called "operators" and _kube-fledged_ to a separate namespace called "kube-fledged". If you need to deploy to a different namespace, export the variables OPERATOR_NAMESPACE and KUBEFLEDGED_NAMESPACE ``` $ make deploy-using-operator @@ -133,36 +133,44 @@ These instructions will help you build _kube-fledged_ from source and deploy it ``` $ export RELEASE_VERSION= - $ export FLEDGED_IMAGE_REPO=/fledged + $ export CONTROLLER_IMAGE_REPO=docker.io//kubefledged-controller + $ export WEBHOOK_SERVER_IMAGE_REPO=docker.io//kubefledged-webhook-server $ docker login -u -p $ export DOCKER_CLI_EXPERIMENTAL=enabled - $ sudo make install-buildx && sudo make fledged-image + $ make install-buildx && make controller-image && make webhook-server-image ``` ### Deploy _Note:- You need to have 'cluster-admin' privileges to deploy_ -- All manifests required for deploying _kube-fledged_ are present in 'kube-fledged/deploy' directory. Edit "kubefledged-deployment.yaml". +- All manifests required for deploying _kube-fledged_ are present in 'kube-fledged/deploy' directory. +- Edit "kubefledged-deployment-controller.yaml". - Set "image" to "/fledged:" + Set "image" to "/kubefledged-controller:" ``` - image: /fledged: + image: /kubefledged-controller: ``` -- If you pushed the image to a private repository, add 'imagePullSecrets' to the end of "kubefledged-deployment.yaml". Refer to kubernetes documentation on [Specifying ImagePullSecrets on a Pod](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod). The secret should be created in "kube-fledged" namespace. +- If you pushed the image to a private repository, add 'imagePullSecrets' to the end of "kubefledged-deployment-controller.yaml". Refer to kubernetes documentation on [Specifying ImagePullSecrets on a Pod](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod). The secret should be created in "kube-fledged" namespace. ``` serviceAccountName: kubefledged imagePullSecrets: - name: ``` +- Edit "kubefledged-deployment-webhook-server.yaml". + Set "image" to "/kubefledged-webhook-server:" + + ``` + image: /kubefledged-webhook-server: + ``` - Deploy _kube-fledged_ to the cluster ``` - $ make deploy + $ make deploy-using-yaml ``` - Verify if _kube-fledged_ deployed successfully @@ -215,7 +223,7 @@ $ kubectl get imagecaches imagecache1 -n kube-fledged -o json _kube-fledged_ supports both automatic and on-demand refresh of image cache. Auto refresh is enabled using the flag `--image-cache-refresh-frequency:`. To request for an on-demand refresh, run the following command:- ``` -$ kubectl annotate imagecaches imagecache1 -n kube-fledged fledged.k8s.io/refresh-imagecache= +$ kubectl annotate imagecaches imagecache1 -n kube-fledged kubefledged.k8s.io/refresh-imagecache= ``` ### Delete image cache @@ -223,7 +231,7 @@ $ kubectl annotate imagecaches imagecache1 -n kube-fledged fledged.k8s.io/refres Before you could delete the image cache, you need to purge the images in the cache using the following command. This will remove all cached images from the worker nodes. ``` -$ kubectl annotate imagecaches imagecache1 -n kube-fledged fledged.k8s.io/purge-imagecache= +$ kubectl annotate imagecaches imagecache1 -n kube-fledged kubefledged.k8s.io/purge-imagecache= ``` View the status of purging the image cache. If any failures, such images should be removed manually or you could decide to leave the images in the worker nodes. @@ -249,9 +257,9 @@ $ make remove-all (if you deployed using Helm Operator) ## How it works -Kubernetes allows developers to extend the kubernetes api via [Custom Resources](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). _kube-fledged_ defines a custom resource of kind “ImageCache” and implements a custom controller (named _fledged_). _fledged_ does the heavy-lifting for managing image cache. Users can use kubectl commands for creation and deletion of ImageCache resources. +Kubernetes allows developers to extend the kubernetes api via [Custom Resources](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). _kube-fledged_ defines a custom resource of kind “ImageCache” and implements a custom controller (named _kubefledged-controller_). _kubefledged-controller_ does the heavy-lifting for managing image cache. Users can use kubectl commands for creation and deletion of ImageCache resources. -_fledged_ has a built-in image manager routine that is responsible for pulling and deleting images. Images are pulled or deleted using kubernetes jobs. If enabled, image cache is refreshed periodically by the refresh worker. _fledged_ updates the status of image pulls, refreshes and image deletions in the status field of ImageCache resource. +_kubefledged-controller_ has a built-in image manager routine that is responsible for pulling and deleting images. Images are pulled or deleted using kubernetes jobs. If enabled, image cache is refreshed periodically by the refresh worker. _kubefledged-controller_ updates the status of image pulls, refreshes and image deletions in the status field of ImageCache resource. For more detailed description, go through _kube-fledged's_ [design proposal](docs/cluster-image-cache.md). From 5398da81fd8062956fc6d17389542deeb44dbf45 Mon Sep 17 00:00:00 2001 From: bhuvanessr Date: Mon, 1 Jun 2020 12:15:56 +0530 Subject: [PATCH 19/21] Update unit tests --- cmd/controller/app/controller.go | 40 ++------------------------- cmd/controller/app/controller_test.go | 36 ++++++++++++------------ 2 files changed, 21 insertions(+), 55 deletions(-) diff --git a/cmd/controller/app/controller.go b/cmd/controller/app/controller.go index a0d07d80..6fa2fc1f 100644 --- a/cmd/controller/app/controller.go +++ b/cmd/controller/app/controller.go @@ -45,7 +45,6 @@ import ( ) const controllerAgentName = "kubefledged-controller" -const fledgedCacheSpecValidationKey = "kubefledged.k8s.io/cachespecvalidation" const imageCachePurgeAnnotationKey = "kubefledged.k8s.io/purge-imagecache" const imageCacheRefreshAnnotationKey = "kubefledged.k8s.io/refresh-imagecache" @@ -293,16 +292,7 @@ func (c *Controller) enqueueImageCache(workType images.WorkType, old, new interf break } } - if !reflect.DeepEqual(newImageCache.Spec, oldImageCache.Spec) { - if validation, ok := newImageCache.Annotations[fledgedCacheSpecValidationKey]; ok { - if validation == "failed" { - if err := c.removeAnnotation(newImageCache, fledgedCacheSpecValidationKey); err != nil { - glog.Errorf("Error removing Annotation %s from imagecache(%s): %v", fledgedCacheSpecValidationKey, newImageCache.Name, err) - } - return false - } - } - } else { + if reflect.DeepEqual(newImageCache.Spec, oldImageCache.Spec) { return false } case images.ImageCacheDelete: @@ -662,38 +652,12 @@ func (c *Controller) updateImageCacheStatus(imageCache *v1alpha1.ImageCache, sta return err } -func (c *Controller) updateImageCacheSpecAndStatus(imageCache *v1alpha1.ImageCache, spec v1alpha1.ImageCacheSpec, status *v1alpha1.ImageCacheStatus) error { - // NEVER modify objects from the store. It's a read-only, local cache. - // You can use DeepCopy() to make a deep copy of original object and modify this copy - // Or create a copy manually for better performance - imageCacheCopy := imageCache.DeepCopy() - imageCacheCopy.Spec = spec - imageCacheCopy.Status = *status - - if status.Status == v1alpha1.ImageCacheActionStatusFailed && - status.Reason == v1alpha1.ImageCacheReasonCacheSpecValidationFailed { - imageCacheCopy.Annotations = make(map[string]string) - imageCacheCopy.Annotations[fledgedCacheSpecValidationKey] = "failed" - } - - if imageCacheCopy.Status.Status != v1alpha1.ImageCacheActionStatusProcessing { - completionTime := metav1.Now() - imageCacheCopy.Status.CompletionTime = &completionTime - } - // If the CustomResourceSubresources feature gate is not enabled, - // we must use Update instead of UpdateStatus to update the Status block of the ImageCache resource. - // UpdateStatus will not allow changes to the Spec of the resource, - // which is ideal for ensuring nothing other than resource status has been updated. - _, err := c.kubefledgedclientset.FledgedV1alpha1().ImageCaches(imageCache.Namespace).Update(imageCacheCopy) - return err -} - func (c *Controller) removeAnnotation(imageCache *v1alpha1.ImageCache, annotationKey string) error { imageCacheCopy := imageCache.DeepCopy() delete(imageCacheCopy.Annotations, annotationKey) _, err := c.kubefledgedclientset.FledgedV1alpha1().ImageCaches(imageCache.Namespace).Update(imageCacheCopy) if err == nil { - glog.Infof("Annotation %s removed from imagecache(%s)", fledgedCacheSpecValidationKey, imageCache.Name) + glog.Infof("Annotation %s removed from imagecache(%s)", annotationKey, imageCache.Name) } return err } diff --git a/cmd/controller/app/controller_test.go b/cmd/controller/app/controller_test.go index e2fa5eed..d0036e22 100644 --- a/cmd/controller/app/controller_test.go +++ b/cmd/controller/app/controller_test.go @@ -999,28 +999,30 @@ func TestEnqueueImageCache(t *testing.T) { oldImageCache: defaultImageCache, expectedResult: true, }, - { - name: "#9: Update - CacheSpec restoration", - workType: images.ImageCacheUpdate, - oldImageCache: defaultImageCache, - newImageCache: kubefledgedv1alpha1.ImageCache{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-fledged", - Annotations: map[string]string{ - fledgedCacheSpecValidationKey: "failed", + /* + { + name: "#9: Update - CacheSpec restoration", + workType: images.ImageCacheUpdate, + oldImageCache: defaultImageCache, + newImageCache: kubefledgedv1alpha1.ImageCache{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-fledged", + Annotations: map[string]string{ + fledgedCacheSpecValidationKey: "failed", + }, }, - }, - Spec: kubefledgedv1alpha1.ImageCacheSpec{ - CacheSpec: []kubefledgedv1alpha1.CacheSpecImages{ - { - Images: []string{"foo", "bar"}, + Spec: kubefledgedv1alpha1.ImageCacheSpec{ + CacheSpec: []kubefledgedv1alpha1.CacheSpecImages{ + { + Images: []string{"foo", "bar"}, + }, }, }, }, + expectedResult: false, }, - expectedResult: false, - }, + */ { name: "#10: Update - Imagecache refresh. Successful queueing", workType: images.ImageCacheUpdate, From a84b89da191b8f818a7e461051aa27b086d03217 Mon Sep 17 00:00:00 2001 From: bhuvanessr Date: Mon, 1 Jun 2020 13:08:21 +0530 Subject: [PATCH 20/21] make latest-tag --- .travis.yml | 3 ++- Makefile | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cf19dad2..5d055db9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,5 +51,6 @@ jobs: script: - docker login -u ${DOCKERHUB_USER} -p ${DOCKERHUB_PSWD} # BUILD_OUTPUT=--push requests buildx to push the built images to docker hub - - travis_wait 60 make BUILD_OUTPUT=--push release + - travis_wait 60 make BUILD_OUTPUT=--push release + - make latest-tag diff --git a/Makefile b/Makefile index b870c5a9..0e2b3965 100644 --- a/Makefile +++ b/Makefile @@ -159,6 +159,20 @@ operator-image: clean-operator release: install-buildx controller-image webhook-server-image cri-client-image operator-image +latest-tag: + docker pull ${CONTROLLER_IMAGE_REPO}:${RELEASE_VERSION} + docker tag ${CONTROLLER_IMAGE_REPO}:${RELEASE_VERSION} ${CONTROLLER_IMAGE_REPO}:latest + docker push ${CONTROLLER_IMAGE_REPO}:latest + docker pull ${WEBHOOK_SERVER_IMAGE_REPO}:${RELEASE_VERSION} + docker tag ${WEBHOOK_SERVER_IMAGE_REPO}:${RELEASE_VERSION} ${WEBHOOK_SERVER_IMAGE_REPO}:latest + docker push ${WEBHOOK_SERVER_IMAGE_REPO}:latest + docker pull ${CRI_CLIENT_IMAGE_REPO}:${RELEASE_VERSION} + docker tag ${CRI_CLIENT_IMAGE_REPO}:${RELEASE_VERSION} ${CRI_CLIENT_IMAGE_REPO}:latest + docker push ${CRI_CLIENT_IMAGE_REPO}:latest + docker pull ${OPERATOR_IMAGE_REPO}:${RELEASE_VERSION} + docker tag ${OPERATOR_IMAGE_REPO}:${RELEASE_VERSION} ${OPERATOR_IMAGE_REPO}:latest + docker push ${OPERATOR_IMAGE_REPO}:latest + install-buildx: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -docker buildx rm multibuilder From ebc3d4817bc876927c5e9df6bda04708ad64e266 Mon Sep 17 00:00:00 2001 From: bhuvanessr Date: Tue, 2 Jun 2020 10:20:47 +0530 Subject: [PATCH 21/21] fix issues in vwc and latest tag --- Makefile | 22 ++++++++++--------- README.md | 2 +- build/Dockerfile.cri_client | 16 +++++++------- .../templates/service-webhook-server.yaml | 2 +- .../templates/validatingwebhook.yaml | 6 ++--- deploy/kubefledged-validatingwebhook.yaml | 2 +- deploy/webhook-create-signed-cert.sh | 11 +++++++--- pkg/images/image_helpers.go | 3 +++ 8 files changed, 37 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index 0e2b3965..e11600ac 100644 --- a/Makefile +++ b/Makefile @@ -186,7 +186,7 @@ test: deploy-using-yaml: -kubectl apply -f deploy/kubefledged-namespace.yaml - bash deploy/webhook-create-signed-cert.sh --namespace kube-fledged --service kubefledged-webhook-server --secret kubefledged-webhook-server + bash deploy/webhook-create-signed-cert.sh bash deploy/webhook-patch-ca-bundle.sh kubectl apply -f deploy/kubefledged-crd.yaml kubectl apply -f deploy/kubefledged-serviceaccount.yaml @@ -213,7 +213,7 @@ deploy-using-operator: # Deploy kube-fledged to a separate namespace sed -i "s|{{OPERATOR_NAMESPACE}}|${OPERATOR_NAMESPACE}|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml sed -i "s|{{KUBEFLEDGED_NAMESPACE}}|${KUBEFLEDGED_NAMESPACE}|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml - bash deploy/webhook-create-signed-cert.sh --namespace ${KUBEFLEDGED_NAMESPACE} --service kubefledged-webhook-server --secret kubefledged-webhook-server + bash deploy/webhook-create-signed-cert.sh --namespace ${KUBEFLEDGED_NAMESPACE} bash deploy/webhook-patch-ca-bundle.sh kubectl apply -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml @@ -224,27 +224,29 @@ update: kubectl scale deployment kubefledged-webhook-server --replicas=1 -n kube-fledged && sleep 1 kubectl get pods -l app=kubefledged -n kube-fledged -remove: +remove-kubefledged: -kubectl delete -f deploy/kubefledged-namespace.yaml -kubectl delete -f deploy/kubefledged-clusterrolebinding.yaml -kubectl delete -f deploy/kubefledged-clusterrole.yaml -kubectl delete -f deploy/kubefledged-crd.yaml -kubectl delete -f deploy/kubefledged-validatingwebhook.yaml + -git checkout deploy/kubefledged-validatingwebhook.yaml + -git checkout deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml -remove-all: +remove-operator-and-kubefledged: # Remove kubefledged and the namespace -kubectl delete -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml -kubectl delete namespace ${KUBEFLEDGED_NAMESPACE} - -sed -i "s|${KUBEFLEDGED_NAMESPACE}|{{KUBEFLEDGED_NAMESPACE}}|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml - -sed -i "s|${OPERATOR_NAMESPACE}|{{OPERATOR_NAMESPACE}}|g" deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml - # Remove the kubefledged-operator and the namespace + -git checkout deploy/kubefledged-validatingwebhook.yaml + -git checkout deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_v1alpha1_kubefledged_cr.yaml + # Remove the kubefledged operator and the namespace -kubectl delete -f deploy/kubefledged-operator/deploy/operator.yaml -kubectl delete -f deploy/kubefledged-operator/deploy/clusterrole_binding.yaml -kubectl delete -f deploy/kubefledged-operator/deploy/clusterrole.yaml -kubectl delete -f deploy/kubefledged-operator/deploy/service_account.yaml -kubectl delete -f deploy/kubefledged-operator/deploy/crds/charts.helm.k8s.io_kubefledgeds_crd.yaml -kubectl delete namespace ${OPERATOR_NAMESPACE} - -sed -i "s|${OPERATOR_NAMESPACE}|{{OPERATOR_NAMESPACE}}|g" deploy/kubefledged-operator/deploy/operator.yaml - -sed -i "s|${OPERATOR_NAMESPACE}|{{OPERATOR_NAMESPACE}}|g" deploy/kubefledged-operator/deploy/clusterrole_binding.yaml - -sed -i "s|${OPERATOR_NAMESPACE}|{{OPERATOR_NAMESPACE}}|g" deploy/kubefledged-operator/deploy/service_account.yaml + -git checkout deploy/kubefledged-operator/deploy/operator.yaml + -git checkout deploy/kubefledged-operator/deploy/clusterrole_binding.yaml + -git checkout deploy/kubefledged-operator/deploy/service_account.yaml diff --git a/README.md b/README.md index 1b117561..5741f37f 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ _kube-fledged_ provides CRUD APIs to manage the lifecycle of the image cache, an - A functioning kubernetes cluster (v1.16 or above). It could be a simple development cluster like minikube or a large production cluster. - All master and worker nodes having the ["kubernetes.io/hostname"](https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/#kubernetes-io-hostname) label. -- git, make, go, docker engine (>= 19.03) and kubectl installed on a local linux machine. kubectl configured properly to access the cluster with cluster-admin privileges. +- git, make, go, docker engine (>= 19.03), openssl and kubectl installed on a local linux machine. kubectl configured properly to access the cluster with cluster-admin privileges. ## Quick Install using YAML manifests diff --git a/build/Dockerfile.cri_client b/build/Dockerfile.cri_client index ce87e732..9c0fa7f1 100644 --- a/build/Dockerfile.cri_client +++ b/build/Dockerfile.cri_client @@ -20,12 +20,12 @@ RUN apk add --no-cache bash curl openssh-client ARG DOCKER_VERSION ARG CRICTL_VERSION -RUN curl -L -o /tmp/docker-$DOCKER_VERSION.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$DOCKER_VERSION.tgz \ - && tar -xz -C /tmp -f /tmp/docker-$DOCKER_VERSION.tgz \ - && mv /tmp/docker/docker /usr/bin \ - && rm -rf /tmp/docker-$DOCKER_VERSION.tgz /tmp/docker +RUN curl -L -o /tmp/docker-$DOCKER_VERSION.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$DOCKER_VERSION.tgz && \ + tar -xz -C /tmp -f /tmp/docker-$DOCKER_VERSION.tgz && \ + mv /tmp/docker/docker /usr/bin && \ + rm -rf /tmp/docker-$DOCKER_VERSION.tgz /tmp/docker -RUN curl -L -o /tmp/crictl-$CRICTL_VERSION.tgz https://github.com/kubernetes-sigs/cri-tools/releases/download/$CRICTL_VERSION/crictl-$CRICTL_VERSION-linux-amd64.tar.gz \ - && tar -xz -C /tmp -f /tmp/crictl-$CRICTL_VERSION.tgz \ - && mv /tmp/crictl /usr/bin \ - && rm -rf /tmp/crictl-$CRICTL_VERSION.tgz /tmp/crictl +RUN curl -L -o /tmp/crictl-$CRICTL_VERSION.tgz https://github.com/kubernetes-sigs/cri-tools/releases/download/$CRICTL_VERSION/crictl-$CRICTL_VERSION-linux-amd64.tar.gz && \ + tar -xz -C /tmp -f /tmp/crictl-$CRICTL_VERSION.tgz && \ + mv /tmp/crictl /usr/bin && \ + rm -rf /tmp/crictl-$CRICTL_VERSION.tgz /tmp/crictl diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/service-webhook-server.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/service-webhook-server.yaml index 9bc4dc56..d0b4bdb8 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/service-webhook-server.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/service-webhook-server.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "kubefledged.webhookServiceName" . }}-webhook-server + name: kubefledged-webhook-server labels: {{ include "kubefledged.labels" . | nindent 4 }} namespace: {{ .Values.kubefledgedNameSpace }} diff --git a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml index b5105b86..1216d306 100644 --- a/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml +++ b/deploy/kubefledged-operator/helm-charts/kubefledged/templates/validatingwebhook.yaml @@ -5,14 +5,14 @@ metadata: name: {{ include "kubefledged.validatingWebhookName" . }} webhooks: - name: validate-image-cache.kubefledged.k8s.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] timeoutSeconds: 1 failurePolicy: Fail sideEffects: None clientConfig: service: - namespace: kube-fledged - name: {{ include "kubefledged.webhookServiceName" . }}-webhook-server + namespace: {{ .Values.kubefledgedNameSpace }} + name: kubefledged-webhook-server path: "/validate-image-cache" port: {{ .Values.webhookService.port }} caBundle: {{ .Values.validatingWebhookCABundle }} diff --git a/deploy/kubefledged-validatingwebhook.yaml b/deploy/kubefledged-validatingwebhook.yaml index 22e08ea9..bae78071 100644 --- a/deploy/kubefledged-validatingwebhook.yaml +++ b/deploy/kubefledged-validatingwebhook.yaml @@ -4,7 +4,7 @@ metadata: name: kubefledged webhooks: - name: validate-image-cache.kubefledged.k8s.io - admissionReviewVersions: ["v1"] + admissionReviewVersions: ["v1beta1", "v1"] timeoutSeconds: 1 failurePolicy: Fail sideEffects: None diff --git a/deploy/webhook-create-signed-cert.sh b/deploy/webhook-create-signed-cert.sh index 9e082c9f..ea18edeb 100644 --- a/deploy/webhook-create-signed-cert.sh +++ b/deploy/webhook-create-signed-cert.sh @@ -60,15 +60,20 @@ while [[ $# -gt 0 ]]; do shift done -[ -z ${service} ] && service=admission-webhook-example-svc -[ -z ${secret} ] && secret=admission-webhook-example-certs -[ -z ${namespace} ] && namespace=default +[ -z ${service} ] && service=kubefledged-webhook-server +[ -z ${secret} ] && secret=kubefledged-webhook-server +[ -z ${namespace} ] && namespace=kube-fledged if [ ! -x "$(command -v openssl)" ]; then echo "openssl not found" exit 1 fi +if [ ! -x "$(command -v kubectl)" ]; then + echo "kubectl not found" + exit 1 +fi + csrName=${service}.${namespace} tmpdir=$(mktemp -d) echo "creating certs in tmpdir ${tmpdir} " diff --git a/pkg/images/image_helpers.go b/pkg/images/image_helpers.go index cd58785b..030fed8a 100644 --- a/pkg/images/image_helpers.go +++ b/pkg/images/image_helpers.go @@ -232,6 +232,9 @@ func checkIfImageNeedsToBePulled(imagePullPolicy string, image string, node *cor if !strings.Contains(image, ":") && !strings.Contains(image, "@sha") { return true, nil } + if strings.Contains(image, ":latest") { + return true, nil + } imageAlreadyPresent, err := imageAlreadyPresentInNode(image, node) if err != nil { return false, err