From 56fd57e1adfa2b6344403b3c79c934ee5cc42152 Mon Sep 17 00:00:00 2001 From: Yusuke Kuoka Date: Thu, 29 Oct 2020 09:32:37 +0900 Subject: [PATCH] Refactor pkg/controller and fix typo on CRD name --- examples/controller/README.md | 18 +- examples/controller/reconcilation.yaml | 2 +- examples/controller/variant.crds.yaml | 8 +- go.mod | 1 + pkg/controller/api.go | 27 +++ pkg/controller/controller.go | 313 +++++++------------------ pkg/controller/handler.go | 20 ++ pkg/controller/run.go | 150 ++++++++++++ 8 files changed, 297 insertions(+), 242 deletions(-) create mode 100644 pkg/controller/api.go create mode 100644 pkg/controller/handler.go create mode 100644 pkg/controller/run.go diff --git a/examples/controller/README.md b/examples/controller/README.md index 85d4747..6bd40d4 100644 --- a/examples/controller/README.md +++ b/examples/controller/README.md @@ -67,18 +67,18 @@ EOS Within a few seconds, the controller will reconcile your `Resource` by running `variant run apply --env preview --ref abc1234`. You can verify that by tailing controller logs by `kubectl logs`, or browsing the `Reconcilation` object that is created by -the controller to record the reconcilation details: +the controller to record the reconciliation details: ```console -$ kubectl get reconcilation +$ kubectl get reconciliation NAME AGE myresource-2 12m ``` ```console -$ kubectl get -o yaml reconcilation myresource-2 +$ kubectl get -o yaml reconciliation myresource-2 apiVersion: core.variant.run/v1beta1 -kind: Reconcilation +kind: Reconciliation metadata: creationTimestamp: "2020-10-28T12:05:55Z" generation: 1 @@ -114,7 +114,7 @@ EOS ``` ```console -$ kubectl get reconcilation +$ kubectl get reconciliation NAME AGE myresource-2 12m myresource-3 12m @@ -122,7 +122,7 @@ myresource-3 12m ```cnosole apiVersion: core.variant.run/v1beta1urce-3 -kind: Reconcilation +kind: Reconciliation metadata: creationTimestamp: "2020-10-28T12:06:10Z" generation: 1 @@ -146,7 +146,7 @@ Finally, deleting the `Resource` will let `variant` destroy the underlying resou as you've configured: ```console -$ kubectl get reconcilation +$ kubectl get reconciliation NAME AGE myresource-2 19m myresource-3 19m @@ -154,9 +154,9 @@ myresource-4 9s ``` ```console -$ kubectl get reconcilation -o yaml myresource-4 +$ kubectl get reconciliation -o yaml myresource-4 apiVersion: core.variant.run/v1beta1 -kind: Reconcilation +kind: Reconciliation metadata: creationTimestamp: "2020-10-28T12:25:32Z" generation: 1 diff --git a/examples/controller/reconcilation.yaml b/examples/controller/reconcilation.yaml index 20d0c9c..1294f84 100644 --- a/examples/controller/reconcilation.yaml +++ b/examples/controller/reconcilation.yaml @@ -1,5 +1,5 @@ apiVersion: core.variant.run/v1beta1 -kind: Reconcilation +kind: Reconciliation metadata: name: myresource-abc spec: diff --git a/examples/controller/variant.crds.yaml b/examples/controller/variant.crds.yaml index a4c5ec9..967fd24 100644 --- a/examples/controller/variant.crds.yaml +++ b/examples/controller/variant.crds.yaml @@ -17,7 +17,7 @@ spec: apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: - name: reconcilations.core.variant.run + name: reconciliations.core.variant.run spec: group: core.variant.run versions: @@ -25,7 +25,7 @@ spec: served: true storage: true names: - kind: Reconcilation - plural: reconcilations - singular: reconcilation + kind: Reconciliation + plural: reconciliations + singular: reconciliation scope: Namespaced diff --git a/go.mod b/go.mod index b8239f7..38ffbb4 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/AlecAivazis/survey/v2 v2.0.5 github.com/PaesslerAG/jsonpath v0.1.1 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/go-logr/logr v0.1.0 github.com/go-playground/universal-translator v0.17.0 // indirect github.com/google/go-cmp v0.4.0 github.com/google/go-github/v27 v27.0.6 // indirect diff --git a/pkg/controller/api.go b/pkg/controller/api.go new file mode 100644 index 0000000..fe73c0b --- /dev/null +++ b/pkg/controller/api.go @@ -0,0 +1,27 @@ +package controller + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +const ( + coreGroup = "core.variant.run" + coreVersion = "v1beta1" +) + +var ( + reconciliationGroupVersionKind = schema.GroupVersionKind{ + Group: coreGroup, + Version: coreVersion, + Kind: "Reconciliation", + } +) + +func newReconciliation() *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + + obj.SetGroupVersionKind(reconciliationGroupVersionKind) + + return obj +} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 4e46620..708d33e 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -3,282 +3,139 @@ package controller import ( "context" "fmt" - "os" - "strings" - - "github.com/summerwind/whitebox-controller/handler" - "github.com/summerwind/whitebox-controller/reconciler/state" + "github.com/go-logr/logr" "golang.org/x/xerrors" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" - kconfig "sigs.k8s.io/controller-runtime/pkg/client/config" - logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" - "sigs.k8s.io/controller-runtime/pkg/runtime/signals" + "strings" +) - _ "k8s.io/client-go/plugin/pkg/client/auth/azure" - _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" - _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" +type controller struct { + // controllerName is the name of this controller that is shown in the logs + controllerName string - "github.com/summerwind/whitebox-controller/config" - "github.com/summerwind/whitebox-controller/manager" -) + // podName is the hostname of where the controller is running on. + // Stored in Reconciliation objects so that the operator can track which controller in which pod has done the + // reconciliation. + podName string -const ( - EnvPrefix = "VARIANT_CONTROLLER_" -) + // Kubernetes client to be used for querying target objects and managing Reconcilation objects + runtimeClient client.Client -func RunRequested() bool { - for _, env := range os.Environ() { - if strings.HasPrefix(env, EnvPrefix) { - return true - } - } + // Runs `variant run ` and returns combined output and/or error + run func([]string) (string, error) - return false + log logr.Logger } -const ( - coreGroup = "core.variant.run" - coreVersion = "v1beta1" -) +func (c *controller) do(job string, obj *unstructured.Unstructured) error { + args := strings.Split(job, " ") -var ( - reconcilationGroupVersionKind = schema.GroupVersionKind{ - Group: coreGroup, - Version: coreVersion, - Kind: "Reconcilation", + m, found, err := unstructured.NestedMap(obj.Object, "spec") + if !found { + return fmt.Errorf(`"spec" field not found: %v`, obj.Object) } -) - -func Run(run func([]string) (string, error)) (finalErr error) { - logf.SetLogger(logf.ZapLogger(false)) - - defer func() { - if finalErr != nil { - logf.Log.Error(finalErr, "Error while running controller") - } - }() - kc, err := kconfig.GetConfig() if err != nil { - return xerrors.Errorf("getting kubernetes client config: %w", err) + return xerrors.Errorf("getting nested map from the object: %w", err) } - getEnv := func(n string) (string, string) { - name := EnvPrefix + n - value := os.Getenv(name) - - return name, value + for k, v := range m { + args = append(args, "--"+k, fmt.Sprintf("%v", v)) } - controllerNameEnv, controllerName := getEnv("NAME") - if controllerName == "" { - return fmt.Errorf("missing required environment variable: %s", controllerNameEnv) - } + c.log.Info("Running Variant", "args", strings.Join(args, " ")) - log := logf.Log.WithName(controllerName) + args = append([]string{"run"}, args...) - _, forAPIVersion := getEnv("FOR_API_VERSION") - if forAPIVersion == "" { - forAPIVersion = coreGroup + "/" + coreVersion + combinedLogs, err := c.run(args) + if err != nil { + return xerrors.Errorf("executing %v: %w", args, err) } - _, forKind := getEnv("FOR_KIND") - if forKind == "" { - forKind = "Resource" + if err := c.logReconciliation(obj, job, combinedLogs); err != nil { + return xerrors.Errorf("logging result of %q: %w", job, err) } - _, resyncPeriod := getEnv("RESYNC_PERIOD") + return nil +} - groupVersion := strings.Split(forAPIVersion, "/") - group := groupVersion[0] - version := groupVersion[1] +func (c *controller) logReconciliation(orig *unstructured.Unstructured, job, combinedLogs string) error { + name := orig.GetName() + namespace := orig.GetNamespace() - jobOnApplyEnv, jobOnApply := getEnv("JOB_ON_APPLY") - if jobOnApply == "" { - return fmt.Errorf("missing required environment variable: %s", jobOnApplyEnv) - } + st := &unstructured.Unstructured{} + st.SetGroupVersionKind(orig.GroupVersionKind()) - jobOnDestroyEnv, jobOnDestroy := getEnv("JOB_ON_DESTROY") - if jobOnDestroy == "" { - return fmt.Errorf("missing required environment variable: %s", jobOnDestroyEnv) + if err := c.runtimeClient.Get(context.TODO(), client.ObjectKey{Namespace: namespace, Name: name}, st); err != nil { + return fmt.Errorf("getting object %q: %w", name, err) } - var runtimeClient client.Client - - podName, err := os.Hostname() - if err != nil { - return xerrors.Errorf("getting pod name from hostname: %w", err) + gen, ok, err := unstructured.NestedInt64(st.Object, "metadata", "generation") + if !ok { + return fmt.Errorf("missing Resource.Generation: %w", err) } - logReconcilation := func(orig *unstructured.Unstructured, job, combinedLogs string) error { - name := orig.GetName() - namespace := orig.GetNamespace() - - st := &unstructured.Unstructured{} - st.SetGroupVersionKind(orig.GroupVersionKind()) - - if err := runtimeClient.Get(context.TODO(), client.ObjectKey{Namespace: namespace, Name: name}, st); err != nil { - return fmt.Errorf("getting object %q: %w", name, err) - } - - gen, ok, err := unstructured.NestedInt64(st.Object, "metadata", "generation") - if !ok { - return fmt.Errorf("missing Resource.Generation: %w", err) - } - - if err != nil { - return xerrors.Errorf("getting metadata.generation from %s: %w", name, err) - } - - reconName := name + "-" + fmt.Sprintf("%d", gen) - - obj := newReconcilation() - - var update bool - - getErr := runtimeClient.Get(context.TODO(), client.ObjectKey{Namespace: namespace, Name: reconName}, obj) - if getErr != nil { - if !errors.IsNotFound(getErr) { - return fmt.Errorf("getting reconcilation object: %w", err) - } - } else { - update = true - } - - // Use of GenerateName results in 404 - //obj.SetGenerateName(name + "-") - obj.SetName(reconName) - // Missing Namespace results in 404 - obj.SetNamespace(namespace) - obj.SetLabels(map[string]string{ - "core.variant.run/event": "apply", - "core.variant.run/controller": controllerName, - "core.variant.run/pod": podName, - }) - spec, ok, err := unstructured.NestedMap(st.Object, "spec") - if !ok { - return fmt.Errorf("missing Resource.Spec: %w", err) - } - - if err != nil { - return xerrors.Errorf("calling unstructured.NestedMap: %w", err) - } - - unstructured.SetNestedField(obj.Object, job, "spec", "job") - unstructured.SetNestedMap(obj.Object, spec, "spec", "resource") - unstructured.SetNestedField(obj.Object, combinedLogs, "spec", "combinedLogs", "data") - - if update { - if err := runtimeClient.Update(context.TODO(), obj); err != nil { - return fmt.Errorf("updating reconcilation object: %w", err) - } - } else { - if err := runtimeClient.Create(context.TODO(), obj); err != nil { - return fmt.Errorf("creating reconcilation object: %w", err) - } - } - - return nil + if err != nil { + return xerrors.Errorf("getting metadata.generation from %s: %w", name, err) } - handle := func(st *state.State, job string) (finalErr error) { - args := strings.Split(job, " ") - m, found, err := unstructured.NestedMap(st.Object.Object, "spec") - if !found { - return fmt.Errorf(`"spec" field not found: %v`, st.Object.Object) - } + reconName := name + "-" + fmt.Sprintf("%d", gen) - if err != nil { - return xerrors.Errorf("getting nested map from the object: %w", err) - } + obj := newReconciliation() - for k, v := range m { - args = append(args, "--"+k, fmt.Sprintf("%v", v)) - } - - log.Info("Running Variant", "args", strings.Join(args, " ")) - - args = append([]string{"run"}, args...) - - combinedLogs, err := run(args) - if err != nil { - return xerrors.Errorf("executing %v: %w", args, err) - } + var update bool - if err := logReconcilation(st.Object, job, combinedLogs); err != nil { - return xerrors.Errorf("logging result of %q: %w", job, err) + getErr := c.runtimeClient.Get(context.TODO(), client.ObjectKey{Namespace: namespace, Name: reconName}, obj) + if getErr != nil { + if !errors.IsNotFound(getErr) { + return fmt.Errorf("getting reconciliation object: %w", err) } - - return nil - } - - applyHandler := StateHandlerFunc(func(st *state.State) error { - return handle(st, jobOnApply) + } else { + update = true + } + + // Use of GenerateName results in 404 + //obj.SetGenerateName(name + "-") + obj.SetName(reconName) + // Missing Namespace results in 404 + obj.SetNamespace(namespace) + obj.SetLabels(map[string]string{ + "core.variant.run/event": "apply", + "core.variant.run/controller": c.controllerName, + "core.variant.run/pod": c.podName, }) - - destroyHandler := StateHandlerFunc(func(st *state.State) error { - return handle(st, jobOnDestroy) - }) - - c := &config.Config{ - Name: controllerName, - Resources: []*config.ResourceConfig{ - { - GroupVersionKind: schema.GroupVersionKind{ - Group: group, - Version: version, - Kind: forKind, - }, - Reconciler: &config.ReconcilerConfig{ - HandlerConfig: config.HandlerConfig{ - StateHandler: applyHandler, - }, - }, - Finalizer: &config.HandlerConfig{ - StateHandler: destroyHandler, - }, - ResyncPeriod: resyncPeriod, - }, - }, - Webhook: nil, + spec, ok, err := unstructured.NestedMap(st.Object, "spec") + if !ok { + return fmt.Errorf("missing Resource.Spec: %w", err) } - mgr, err := manager.New(c, kc) if err != nil { - return xerrors.Errorf("creating controller-manager: %w", err) + return xerrors.Errorf("calling unstructured.NestedMap: %w", err) } - runtimeClient = mgr.GetClient() - - err = mgr.Start(signals.SetupSignalHandler()) - if err != nil { - return xerrors.Errorf("starting controller-manager: %w", err) + if err := unstructured.SetNestedField(obj.Object, job, "spec", "job"); err != nil { + return xerrors.Errorf("setting nested field spec.job: %w", err) } - return err -} - -func newReconcilation() *unstructured.Unstructured { - obj := &unstructured.Unstructured{} - - obj.SetGroupVersionKind(reconcilationGroupVersionKind) - - return obj -} + if err := unstructured.SetNestedMap(obj.Object, spec, "spec", "resource"); err != nil { + return xerrors.Errorf("setting nested map spec.resource: %w", err) + } -func StateHandlerFunc(f func(*state.State) error) handler.StateHandler { - return &stateHandler{ - f: f, + if err := unstructured.SetNestedField(obj.Object, combinedLogs, "spec", "combinedLogs", "data"); err != nil { + return xerrors.Errorf("setting nested field spec.combinedLogs.data: %w", err) } -} -type stateHandler struct { - f func(*state.State) error -} + if update { + if err := c.runtimeClient.Update(context.TODO(), obj); err != nil { + return fmt.Errorf("updating reconciliation object: %w", err) + } + } else { + if err := c.runtimeClient.Create(context.TODO(), obj); err != nil { + return fmt.Errorf("creating reconciliation object: %w", err) + } + } -func (h stateHandler) HandleState(s *state.State) error { - return h.f(s) + return nil } diff --git a/pkg/controller/handler.go b/pkg/controller/handler.go new file mode 100644 index 0000000..93cea75 --- /dev/null +++ b/pkg/controller/handler.go @@ -0,0 +1,20 @@ +package controller + +import ( + "github.com/summerwind/whitebox-controller/handler" + "github.com/summerwind/whitebox-controller/reconciler/state" +) + +func StateHandlerFunc(f func(*state.State) error) handler.StateHandler { + return &stateHandler{ + f: f, + } +} + +type stateHandler struct { + f func(*state.State) error +} + +func (h stateHandler) HandleState(s *state.State) error { + return h.f(s) +} diff --git a/pkg/controller/run.go b/pkg/controller/run.go new file mode 100644 index 0000000..58a1e33 --- /dev/null +++ b/pkg/controller/run.go @@ -0,0 +1,150 @@ +package controller + +import ( + "fmt" + "os" + "strings" + + "github.com/summerwind/whitebox-controller/reconciler/state" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/runtime/schema" + kconfig "sigs.k8s.io/controller-runtime/pkg/client/config" + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/runtime/signals" + + _ "k8s.io/client-go/plugin/pkg/client/auth/azure" + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" + + "github.com/summerwind/whitebox-controller/config" + "github.com/summerwind/whitebox-controller/manager" +) + +const ( + EnvPrefix = "VARIANT_CONTROLLER_" +) + +func RunRequested() bool { + for _, env := range os.Environ() { + if strings.HasPrefix(env, EnvPrefix) { + return true + } + } + + return false +} + +func Run(run func([]string) (string, error)) (finalErr error) { + logf.SetLogger(logf.ZapLogger(false)) + + defer func() { + if finalErr != nil { + logf.Log.Error(finalErr, "Error while running controller") + } + }() + + kc, err := kconfig.GetConfig() + if err != nil { + return xerrors.Errorf("getting kubernetes client config: %w", err) + } + + getEnv := func(n string) (string, string) { + name := EnvPrefix + n + value := os.Getenv(name) + + return name, value + } + + controllerNameEnv, controllerName := getEnv("NAME") + if controllerName == "" { + return fmt.Errorf("missing required environment variable: %s", controllerNameEnv) + } + + _, forAPIVersion := getEnv("FOR_API_VERSION") + if forAPIVersion == "" { + forAPIVersion = coreGroup + "/" + coreVersion + } + + _, forKind := getEnv("FOR_KIND") + if forKind == "" { + forKind = "Resource" + } + + _, resyncPeriod := getEnv("RESYNC_PERIOD") + + groupVersion := strings.Split(forAPIVersion, "/") + group := groupVersion[0] + version := groupVersion[1] + + jobOnApplyEnv, jobOnApply := getEnv("JOB_ON_APPLY") + if jobOnApply == "" { + return fmt.Errorf("missing required environment variable: %s", jobOnApplyEnv) + } + + jobOnDestroyEnv, jobOnDestroy := getEnv("JOB_ON_DESTROY") + if jobOnDestroy == "" { + return fmt.Errorf("missing required environment variable: %s", jobOnDestroyEnv) + } + + podName, err := os.Hostname() + if err != nil { + return xerrors.Errorf("getting pod name from hostname: %w", err) + } + + ctl := &controller{ + log: logf.Log.WithName(controllerName), + runtimeClient: nil, + run: run, + podName: podName, + controllerName: controllerName, + } + + handle := func(st *state.State, job string) (finalErr error) { + return ctl.do(job, st.Object) + } + + applyHandler := StateHandlerFunc(func(st *state.State) error { + return handle(st, jobOnApply) + }) + + destroyHandler := StateHandlerFunc(func(st *state.State) error { + return handle(st, jobOnDestroy) + }) + + c := &config.Config{ + Name: controllerName, + Resources: []*config.ResourceConfig{ + { + GroupVersionKind: schema.GroupVersionKind{ + Group: group, + Version: version, + Kind: forKind, + }, + Reconciler: &config.ReconcilerConfig{ + HandlerConfig: config.HandlerConfig{ + StateHandler: applyHandler, + }, + }, + Finalizer: &config.HandlerConfig{ + StateHandler: destroyHandler, + }, + ResyncPeriod: resyncPeriod, + }, + }, + Webhook: nil, + } + + mgr, err := manager.New(c, kc) + if err != nil { + return xerrors.Errorf("creating controller-manager: %w", err) + } + + ctl.runtimeClient = mgr.GetClient() + + err = mgr.Start(signals.SetupSignalHandler()) + if err != nil { + return xerrors.Errorf("starting controller-manager: %w", err) + } + + return err +}