From 04fcf1a922cc6669de8209c42bb2dbab8e584001 Mon Sep 17 00:00:00 2001 From: viovanov Date: Tue, 11 Dec 2018 07:45:16 +0200 Subject: [PATCH 1/2] Initial implementation for the ExtendedStatefulSet --- Makefile | 6 +- integration/environment/environment.go | 27 +++++- integration/environment/machine.go | 117 +++++++++++++++++++------ pkg/kube/controllers/controllers.go | 12 +-- 4 files changed, 124 insertions(+), 38 deletions(-) diff --git a/Makefile b/Makefile index 8dc8bcd8..2d2da796 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ helm: export CFO_NAMESPACE ?= default up: kubectl apply -f deploy/helm/cf-operator/templates/fissile_v1alpha1_boshdeployment_crd.yaml + kubectl apply -f deploy/helm/cf-operator/templates/fissile_v1alpha1_extendedstatefulset_crd.yaml @echo watching namespace ${CFO_NAMESPACE} go run cmd/cf-operator/main.go @@ -39,7 +40,10 @@ test-unit: test-integration: bin/test-integration -test: vet lint test-unit test-integration +test-e2e: + bin/test-integration + +test: vet lint test-unit test-integration test-e2e tools: bin/tools \ No newline at end of file diff --git a/integration/environment/environment.go b/integration/environment/environment.go index fca48075..453f4dff 100644 --- a/integration/environment/environment.go +++ b/integration/environment/environment.go @@ -1,6 +1,7 @@ package environment import ( + "fmt" "log" "os" "path/filepath" @@ -68,6 +69,11 @@ func (e *Environment) Setup() (StopFunc, error) { }, nil } +// FlushLog flushes the zap log +func (e *Environment) FlushLog() error { + return e.log.Sync() +} + // AllLogMessages returns only the message part of existing logs to aid in debugging func (e *Environment) AllLogMessages() (msgs []string) { for _, m := range e.LogRecorded.All() { @@ -84,9 +90,24 @@ func (e *Environment) setupCFOperator() (err error) { } e.Namespace = ns - var core zapcore.Core - core, e.LogRecorded = observer.New(zapcore.DebugLevel) - e.log = zap.New(core).Sugar() + // An in-memory zap core that can be used for assertions + var memCore zapcore.Core + memCore, e.LogRecorded = observer.New(zapcore.DebugLevel) + + // A zap core that writes to a temp file + consoleEncoder := zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()) + f, err := os.Create("/tmp/cf-operator-tests.log") + if err != nil { + panic(fmt.Sprintf("can't create log file: %s\n", err.Error())) + } + fileCore := zapcore.NewCore( + consoleEncoder, + zapcore.Lock(f), + zap.LevelEnablerFunc(func(lvl zapcore.Level) bool { + return true + })) + + e.log = zap.New(zapcore.NewTee(memCore, fileCore)).Sugar() err = e.setupKube() if err != nil { diff --git a/integration/environment/machine.go b/integration/environment/machine.go index 33aa79bf..6e2ac7ba 100644 --- a/integration/environment/machine.go +++ b/integration/environment/machine.go @@ -1,19 +1,19 @@ package environment import ( - "fmt" "time" - bdcv1 "code.cloudfoundry.org/cf-operator/pkg/kube/apis/boshdeployment/v1alpha1" - "code.cloudfoundry.org/cf-operator/pkg/kube/client/clientset/versioned" "github.com/pkg/errors" apiv1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" + + bdcv1 "code.cloudfoundry.org/cf-operator/pkg/kube/apis/boshdeploymentcontroller/v1alpha1" + essv1 "code.cloudfoundry.org/cf-operator/pkg/kube/apis/extendedstatefulsetcontroller/v1alpha1" + "code.cloudfoundry.org/cf-operator/pkg/kube/client/clientset/versioned" ) // Machine produces and destroys resources for tests @@ -35,6 +35,33 @@ func (m *Machine) WaitForPod(namespace string, name string) error { }) } +// WaitForPods blocks until all selected pods are running. It fails after the timeout. +func (m *Machine) WaitForPods(namespace string, labels string) error { + return wait.PollImmediate(m.pollInterval, m.pollTimeout, func() (bool, error) { + return m.PodsRunning(namespace, labels) + }) +} + +// WaitForExtendedStatefulSets blocks until at least one WaitForExtendedStatefulSet is found. It fails after the timeout. +func (m *Machine) WaitForExtendedStatefulSets(namespace string, labels string) error { + return wait.PollImmediate(m.pollInterval, m.pollTimeout, func() (bool, error) { + return m.ExtendedStatefulSetExists(namespace, labels) + }) +} + +// ExtendedStatefulSetExists returns true if at least one ess selected by labels exists +func (m *Machine) ExtendedStatefulSetExists(namespace string, labels string) (bool, error) { + esss, err := m.VersionedClientset.ExtendedstatefulsetcontrollerV1alpha1().ExtendedStatefulSets(namespace).List(v1.ListOptions{ + LabelSelector: labels, + }) + if err != nil { + return false, errors.Wrapf(err, "failed to query for ess by labels: %v", labels) + } + + return len(esss.Items) > 0, nil +} + + // WaitForPodsDelete blocks until the pod is deleted. It fails after the timeout. func (m *Machine) WaitForPodsDelete(namespace string) error { return wait.PollImmediate(m.pollInterval, m.pollTimeout, func() (bool, error) { @@ -70,33 +97,41 @@ func (m *Machine) PodRunning(namespace string, name string) (bool, error) { return false, nil } -// PodLabeled returns true if the pod is interpolated correctly -func (m *Machine) PodLabeled(namespace string, name string, desiredLabel, desiredValue string) (bool, error) { - pod, err := m.Clientset.CoreV1().Pods(namespace).Get(name, v1.GetOptions{}) +// PodsRunning returns true if all the pods selected by labels are in state running +// Note that only the first page of pods is considered - don't use this if you have a +// long pod list that you care about +func (m *Machine) PodsRunning(namespace string, labels string) (bool, error) { + pods, err := m.Clientset.CoreV1().Pods(namespace).List(v1.ListOptions{ + LabelSelector: labels, + }) if err != nil { - if apierrors.IsNotFound(err) { - return false, err - } - return false, errors.Wrapf(err, "Failed to query for pod by name: %s", name) + return false, errors.Wrapf(err, "failed to query for pod by labels: %v", labels) } - if pod.ObjectMeta.Labels[desiredLabel] == desiredValue { - return true, nil + if len(pods.Items) == 0 { + return false, nil + } + + for _, pod := range pods.Items { + if pod.Status.Phase != apiv1.PodRunning { + return false, nil + } } - return false, fmt.Errorf("Cannot match the desired label with %s", desiredValue) + + return true, nil } -// WaitForCRDeletion blocks until the CR is deleted -func (m *Machine) WaitForCRDeletion(namespace string, name string) error { +// WaitForBOSHDeploymentDeletion blocks until the CR is deleted +func (m *Machine) WaitForBOSHDeploymentDeletion(namespace string, name string) error { return wait.PollImmediate(m.pollInterval, m.pollTimeout, func() (bool, error) { - found, err := m.HasFissileCR(namespace, name) + found, err := m.HasBOSHDeployment(namespace, name) return !found, err }) } -// HasFissileCR returns true if the pod by that name is in state running -func (m *Machine) HasFissileCR(namespace string, name string) (bool, error) { - client := m.VersionedClientset.Boshdeployment().BOSHDeployments(namespace) +// HasBOSHDeployment returns true if the pod by that name is in state running +func (m *Machine) HasBOSHDeployment(namespace string, name string) (bool, error) { + client := m.VersionedClientset.Boshdeploymentcontroller().BOSHDeployments(namespace) _, err := client.Get(name, v1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { @@ -126,26 +161,50 @@ func (m *Machine) CreateSecret(namespace string, secret corev1.Secret) (TearDown }, err } -// CreateFissileCR creates a BOSHDeployment custom resource and returns a function to delete it -func (m *Machine) CreateFissileCR(namespace string, deployment bdcv1.BOSHDeployment) (*bdcv1.BOSHDeployment, TearDownFunc, error) { - client := m.VersionedClientset.Boshdeployment().BOSHDeployments(namespace) +// CreateBOSHDeployment creates a BOSHDeployment custom resource and returns a function to delete it +func (m *Machine) CreateBOSHDeployment(namespace string, deployment bdcv1.BOSHDeployment) (*bdcv1.BOSHDeployment, TearDownFunc, error) { + client := m.VersionedClientset.Boshdeploymentcontroller().BOSHDeployments(namespace) d, err := client.Create(&deployment) return d, func() { client.Delete(deployment.GetName(), &v1.DeleteOptions{}) }, err } -// UpdateFissileCR creates a BOSHDeployment custom resource and returns a function to delete it -func (m *Machine) UpdateFissileCR(namespace string, deployment bdcv1.BOSHDeployment) (*bdcv1.BOSHDeployment, TearDownFunc, error) { - client := m.VersionedClientset.Boshdeployment().BOSHDeployments(namespace) +// UpdateBOSHDeployment creates a BOSHDeployment custom resource and returns a function to delete it +func (m *Machine) UpdateBOSHDeployment(namespace string, deployment bdcv1.BOSHDeployment) (*bdcv1.BOSHDeployment, TearDownFunc, error) { + client := m.VersionedClientset.Boshdeploymentcontroller().BOSHDeployments(namespace) d, err := client.Update(&deployment) return d, func() { client.Delete(deployment.GetName(), &v1.DeleteOptions{}) }, err } -// DeleteFissileCR deletes a BOSHDeployment custom resource -func (m *Machine) DeleteFissileCR(namespace string, name string) error { - client := m.VersionedClientset.Boshdeployment().BOSHDeployments(namespace) +// DeleteBOSHDeployment deletes a BOSHDeployment custom resource +func (m *Machine) DeleteBOSHDeployment(namespace string, name string) error { + client := m.VersionedClientset.Boshdeploymentcontroller().BOSHDeployments(namespace) + return client.Delete(name, &v1.DeleteOptions{}) +} + +// CreateExtendedStatefulSet creates a ExtendedStatefulSet custom resource and returns a function to delete it +func (m *Machine) CreateExtendedStatefulSet(namespace string, ess essv1.ExtendedStatefulSet) (*essv1.ExtendedStatefulSet, TearDownFunc, error) { + client := m.VersionedClientset.Extendedstatefulsetcontroller().ExtendedStatefulSets(namespace) + d, err := client.Create(&ess) + return d, func() { + client.Delete(ess.GetName(), &v1.DeleteOptions{}) + }, err +} + +// UpdateExtendedStatefulSet creates a ExtendedStatefulSet custom resource and returns a function to delete it +func (m *Machine) UpdateExtendedStatefulSet(namespace string, ess essv1.ExtendedStatefulSet) (*essv1.ExtendedStatefulSet, TearDownFunc, error) { + client := m.VersionedClientset.Extendedstatefulsetcontroller().ExtendedStatefulSets(namespace) + d, err := client.Update(&ess) + return d, func() { + client.Delete(ess.GetName(), &v1.DeleteOptions{}) + }, err +} + +// DeleteExtendedStatefulSet deletes a ExtendedStatefulSet custom resource +func (m *Machine) DeleteExtendedStatefulSet(namespace string, name string) error { + client := m.VersionedClientset.Extendedstatefulsetcontroller().ExtendedStatefulSets(namespace) return client.Delete(name, &v1.DeleteOptions{}) } diff --git a/pkg/kube/controllers/controllers.go b/pkg/kube/controllers/controllers.go index 9e07ebc0..492883bf 100644 --- a/pkg/kube/controllers/controllers.go +++ b/pkg/kube/controllers/controllers.go @@ -1,22 +1,24 @@ -package controllers +package controller import ( "go.uber.org/zap" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/manager" - bdc "code.cloudfoundry.org/cf-operator/pkg/kube/apis/boshdeployment/v1alpha1" - ess "code.cloudfoundry.org/cf-operator/pkg/kube/apis/extendedstatefulset/v1alpha1" + bdcv1 "code.cloudfoundry.org/cf-operator/pkg/kube/apis/boshdeployment/v1alpha1" + essv1 "code.cloudfoundry.org/cf-operator/pkg/kube/apis/extendedstatefulset/v1alpha1" "code.cloudfoundry.org/cf-operator/pkg/kube/controllers/boshdeployment" + "code.cloudfoundry.org/cf-operator/pkg/kube/controllers/extendedstatefulset" ) var addToManagerFuncs = []func(*zap.SugaredLogger, manager.Manager) error{ boshdeployment.Add, + extendedstatefulset.Add, } var addToSchemes = runtime.SchemeBuilder{ - bdc.AddToScheme, - ess.AddToScheme, + bdcv1.AddToScheme, + essv1.AddToScheme, } // AddToManager adds all Controllers to the Manager From f8df5ffd1fd7ee5eceba918c95d63a493a89c174 Mon Sep 17 00:00:00 2001 From: viovanov Date: Tue, 11 Dec 2018 11:59:03 +0200 Subject: [PATCH 2/2] Fixes after rebase --- Makefile | 4 +- integration/environment/machine.go | 65 ++++++++++++++++++----------- pkg/kube/controllers/controllers.go | 2 +- 3 files changed, 43 insertions(+), 28 deletions(-) diff --git a/Makefile b/Makefile index 2d2da796..410bebfa 100644 --- a/Makefile +++ b/Makefile @@ -41,9 +41,9 @@ test-integration: bin/test-integration test-e2e: - bin/test-integration + bin/test-e2e test: vet lint test-unit test-integration test-e2e tools: - bin/tools \ No newline at end of file + bin/tools diff --git a/integration/environment/machine.go b/integration/environment/machine.go index 6e2ac7ba..c2b37545 100644 --- a/integration/environment/machine.go +++ b/integration/environment/machine.go @@ -1,18 +1,18 @@ package environment import ( + "fmt" "time" "github.com/pkg/errors" - apiv1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" - bdcv1 "code.cloudfoundry.org/cf-operator/pkg/kube/apis/boshdeploymentcontroller/v1alpha1" - essv1 "code.cloudfoundry.org/cf-operator/pkg/kube/apis/extendedstatefulsetcontroller/v1alpha1" + bdcv1 "code.cloudfoundry.org/cf-operator/pkg/kube/apis/boshdeployment/v1alpha1" + essv1 "code.cloudfoundry.org/cf-operator/pkg/kube/apis/extendedstatefulset/v1alpha1" "code.cloudfoundry.org/cf-operator/pkg/kube/client/clientset/versioned" ) @@ -51,7 +51,7 @@ func (m *Machine) WaitForExtendedStatefulSets(namespace string, labels string) e // ExtendedStatefulSetExists returns true if at least one ess selected by labels exists func (m *Machine) ExtendedStatefulSetExists(namespace string, labels string) (bool, error) { - esss, err := m.VersionedClientset.ExtendedstatefulsetcontrollerV1alpha1().ExtendedStatefulSets(namespace).List(v1.ListOptions{ + esss, err := m.VersionedClientset.ExtendedstatefulsetV1alpha1().ExtendedStatefulSets(namespace).List(metav1.ListOptions{ LabelSelector: labels, }) if err != nil { @@ -61,7 +61,6 @@ func (m *Machine) ExtendedStatefulSetExists(namespace string, labels string) (bo return len(esss.Items) > 0, nil } - // WaitForPodsDelete blocks until the pod is deleted. It fails after the timeout. func (m *Machine) WaitForPodsDelete(namespace string) error { return wait.PollImmediate(m.pollInterval, m.pollTimeout, func() (bool, error) { @@ -83,7 +82,7 @@ func (m *Machine) PodsDeleted(namespace string) (bool, error) { // PodRunning returns true if the pod by that name is in state running func (m *Machine) PodRunning(namespace string, name string) (bool, error) { - pod, err := m.Clientset.CoreV1().Pods(namespace).Get(name, v1.GetOptions{}) + pod, err := m.Clientset.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { return false, nil @@ -91,7 +90,7 @@ func (m *Machine) PodRunning(namespace string, name string) (bool, error) { return false, errors.Wrapf(err, "failed to query for pod by name: %s", name) } - if pod.Status.Phase == apiv1.PodRunning { + if pod.Status.Phase == corev1.PodRunning { return true, nil } return false, nil @@ -101,7 +100,7 @@ func (m *Machine) PodRunning(namespace string, name string) (bool, error) { // Note that only the first page of pods is considered - don't use this if you have a // long pod list that you care about func (m *Machine) PodsRunning(namespace string, labels string) (bool, error) { - pods, err := m.Clientset.CoreV1().Pods(namespace).List(v1.ListOptions{ + pods, err := m.Clientset.CoreV1().Pods(namespace).List(metav1.ListOptions{ LabelSelector: labels, }) if err != nil { @@ -113,7 +112,7 @@ func (m *Machine) PodsRunning(namespace string, labels string) (bool, error) { } for _, pod := range pods.Items { - if pod.Status.Phase != apiv1.PodRunning { + if pod.Status.Phase != corev1.PodRunning { return false, nil } } @@ -131,8 +130,8 @@ func (m *Machine) WaitForBOSHDeploymentDeletion(namespace string, name string) e // HasBOSHDeployment returns true if the pod by that name is in state running func (m *Machine) HasBOSHDeployment(namespace string, name string) (bool, error) { - client := m.VersionedClientset.Boshdeploymentcontroller().BOSHDeployments(namespace) - _, err := client.Get(name, v1.GetOptions{}) + client := m.VersionedClientset.Boshdeployment().BOSHDeployments(namespace) + _, err := client.Get(name, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { return false, nil @@ -148,7 +147,7 @@ func (m *Machine) CreateConfigMap(namespace string, configMap corev1.ConfigMap) client := m.Clientset.CoreV1().ConfigMaps(namespace) _, err := client.Create(&configMap) return func() { - client.Delete(configMap.GetName(), &v1.DeleteOptions{}) + client.Delete(configMap.GetName(), &metav1.DeleteOptions{}) }, err } @@ -157,54 +156,70 @@ func (m *Machine) CreateSecret(namespace string, secret corev1.Secret) (TearDown client := m.Clientset.CoreV1().Secrets(namespace) _, err := client.Create(&secret) return func() { - client.Delete(secret.GetName(), &v1.DeleteOptions{}) + client.Delete(secret.GetName(), &metav1.DeleteOptions{}) }, err } // CreateBOSHDeployment creates a BOSHDeployment custom resource and returns a function to delete it func (m *Machine) CreateBOSHDeployment(namespace string, deployment bdcv1.BOSHDeployment) (*bdcv1.BOSHDeployment, TearDownFunc, error) { - client := m.VersionedClientset.Boshdeploymentcontroller().BOSHDeployments(namespace) + client := m.VersionedClientset.Boshdeployment().BOSHDeployments(namespace) d, err := client.Create(&deployment) return d, func() { - client.Delete(deployment.GetName(), &v1.DeleteOptions{}) + client.Delete(deployment.GetName(), &metav1.DeleteOptions{}) }, err } // UpdateBOSHDeployment creates a BOSHDeployment custom resource and returns a function to delete it func (m *Machine) UpdateBOSHDeployment(namespace string, deployment bdcv1.BOSHDeployment) (*bdcv1.BOSHDeployment, TearDownFunc, error) { - client := m.VersionedClientset.Boshdeploymentcontroller().BOSHDeployments(namespace) + client := m.VersionedClientset.BoshdeploymentV1alpha1().BOSHDeployments(namespace) d, err := client.Update(&deployment) return d, func() { - client.Delete(deployment.GetName(), &v1.DeleteOptions{}) + client.Delete(deployment.GetName(), &metav1.DeleteOptions{}) }, err } // DeleteBOSHDeployment deletes a BOSHDeployment custom resource func (m *Machine) DeleteBOSHDeployment(namespace string, name string) error { - client := m.VersionedClientset.Boshdeploymentcontroller().BOSHDeployments(namespace) - return client.Delete(name, &v1.DeleteOptions{}) + client := m.VersionedClientset.BoshdeploymentV1alpha1().BOSHDeployments(namespace) + return client.Delete(name, &metav1.DeleteOptions{}) } // CreateExtendedStatefulSet creates a ExtendedStatefulSet custom resource and returns a function to delete it func (m *Machine) CreateExtendedStatefulSet(namespace string, ess essv1.ExtendedStatefulSet) (*essv1.ExtendedStatefulSet, TearDownFunc, error) { - client := m.VersionedClientset.Extendedstatefulsetcontroller().ExtendedStatefulSets(namespace) + client := m.VersionedClientset.ExtendedstatefulsetV1alpha1().ExtendedStatefulSets(namespace) d, err := client.Create(&ess) return d, func() { - client.Delete(ess.GetName(), &v1.DeleteOptions{}) + client.Delete(ess.GetName(), &metav1.DeleteOptions{}) }, err } // UpdateExtendedStatefulSet creates a ExtendedStatefulSet custom resource and returns a function to delete it func (m *Machine) UpdateExtendedStatefulSet(namespace string, ess essv1.ExtendedStatefulSet) (*essv1.ExtendedStatefulSet, TearDownFunc, error) { - client := m.VersionedClientset.Extendedstatefulsetcontroller().ExtendedStatefulSets(namespace) + client := m.VersionedClientset.ExtendedstatefulsetV1alpha1().ExtendedStatefulSets(namespace) d, err := client.Update(&ess) return d, func() { - client.Delete(ess.GetName(), &v1.DeleteOptions{}) + client.Delete(ess.GetName(), &metav1.DeleteOptions{}) }, err } // DeleteExtendedStatefulSet deletes a ExtendedStatefulSet custom resource func (m *Machine) DeleteExtendedStatefulSet(namespace string, name string) error { - client := m.VersionedClientset.Extendedstatefulsetcontroller().ExtendedStatefulSets(namespace) - return client.Delete(name, &v1.DeleteOptions{}) + client := m.VersionedClientset.ExtendedstatefulsetV1alpha1().ExtendedStatefulSets(namespace) + return client.Delete(name, &metav1.DeleteOptions{}) +} + +// PodLabeled returns true if the pod is labeled correctly +func (m *Machine) PodLabeled(namespace string, name string, desiredLabel, desiredValue string) (bool, error) { + pod, err := m.Clientset.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, err + } + return false, errors.Wrapf(err, "Failed to query for pod by name: %s", name) + } + + if pod.ObjectMeta.Labels[desiredLabel] == desiredValue { + return true, nil + } + return false, fmt.Errorf("Cannot match the desired label with %s", desiredValue) } diff --git a/pkg/kube/controllers/controllers.go b/pkg/kube/controllers/controllers.go index 492883bf..aa5bf7e6 100644 --- a/pkg/kube/controllers/controllers.go +++ b/pkg/kube/controllers/controllers.go @@ -1,4 +1,4 @@ -package controller +package controllers import ( "go.uber.org/zap"