From 4a85ca92d723b374bcd44bf81ff6bcd1d17f4599 Mon Sep 17 00:00:00 2001 From: Guilhem Lettron Date: Fri, 15 Dec 2017 00:03:12 +0100 Subject: [PATCH 01/14] Add option for operator to act as cluster wide for a cluster to be managed widely, user have to add annotation: etcd.database.coreos.com/scope: clusterwide And admin have to run an operator with option "-cluster-wide". Current implementation have a lack of locking if many operators with cluster-wide run on different namespaces. --- README.md | 6 ++- cmd/operator/main.go | 4 ++ doc/user/clusterwide.md | 26 ++++++++++++ example/deployment.yaml | 2 + pkg/controller/controller.go | 5 +++ pkg/controller/controller_test.go | 68 +++++++++++++++++++++++++++++-- pkg/controller/informer.go | 24 ++++++++++- pkg/util/constants/constants.go | 3 ++ 8 files changed, 132 insertions(+), 6 deletions(-) create mode 100644 doc/user/clusterwide.md diff --git a/README.md b/README.md index cb5facffa..0a1b66ac4 100644 --- a/README.md +++ b/README.md @@ -278,9 +278,11 @@ Follow the [etcd backup operator walkthrough](./doc/user/walkthrough/backup-oper Follow the [etcd restore operator walkthrough](./doc/user/walkthrough/restore-operator.md) to restore an etcd cluster on Kubernetes from backup. -### Limitations +### Manage etcd clusters in all namespaces + +See [instructions on clusterwide feature](doc/user/clusterwide.md). -- The etcd operator only manages the etcd cluster created in the same namespace. Users need to create multiple operators in different namespaces to manage etcd clusters in different namespaces. +### Limitations [k8s-home]: http://kubernetes.io diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 6277ef564..7e2cb2fa6 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -57,6 +57,8 @@ var ( printVersion bool createCRD bool + + clusterWide bool ) func init() { @@ -66,6 +68,7 @@ func init() { flag.BoolVar(&printVersion, "version", false, "Show version and quit") flag.BoolVar(&createCRD, "create-crd", true, "The operator will not create the EtcdCluster CRD when this flag is set to false.") flag.DurationVar(&gcInterval, "gc-interval", 10*time.Minute, "GC interval") + flag.BoolVar(&clusterWide, "cluster-wide", false, "The operator will watch clusters in all namespaces") flag.Parse() } @@ -151,6 +154,7 @@ func newControllerConfig() controller.Config { cfg := controller.Config{ Namespace: namespace, + ClusterWide: clusterWide, ServiceAccount: serviceAccount, KubeCli: kubecli, KubeExtCli: k8sutil.MustNewKubeExtClient(), diff --git a/doc/user/clusterwide.md b/doc/user/clusterwide.md new file mode 100644 index 000000000..a4ddaa777 --- /dev/null +++ b/doc/user/clusterwide.md @@ -0,0 +1,26 @@ +# Manage clusters in all namespaces + +Default etcd operator behavior is to only manage etcd clusters created in the same namespace. +It is possible to deploy an etcd operator with special option to manage clusterwide etcd clusters. + +## Install etcd operator + +etcd operator have to run with `-cluster-wide` arg option. + +More information in [install guide](doc/user/install_guide.md). + +## Special annotation + +To declare an etcd cluster as "clusterwide", you have to add special annotation `etcd.database.coreos.com/scope` with value `clusterwide`. + +```yaml +apiVersion: "etcd.database.coreos.com/v1beta2" +kind: "EtcdCluster" +metadata: + name: "example-etcd-cluster" + annotations: + etcd.database.coreos.com/scope: clusterwide +spec: + size: 3 + version: "3.2.13" +``` diff --git a/example/deployment.yaml b/example/deployment.yaml index 0d0fe056c..bd3f5352c 100644 --- a/example/deployment.yaml +++ b/example/deployment.yaml @@ -14,6 +14,8 @@ spec: image: quay.io/coreos/etcd-operator:v0.9.0 command: - etcd-operator + # Uncomment to act for resources in all namespaces. More information in doc/clusterwide.md + #- -cluster-wide env: - name: MY_POD_NAMESPACE valueFrom: diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 0a6b4c64f..17b7b2d7d 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -45,6 +45,7 @@ type Controller struct { type Config struct { Namespace string + ClusterWide bool ServiceAccount string KubeCli kubernetes.Interface KubeExtCli apiextensionsclient.Interface @@ -64,6 +65,10 @@ func New(cfg Config) *Controller { func (c *Controller) handleClusterEvent(event *Event) error { clus := event.Object + if !c.managed(clus) { + return fmt.Errorf("cluster (%s) isn't managed", clus.Name) + } + if clus.Status.IsFailed() { clustersFailed.Inc() if event.Type == kwatch.Deleted { diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index 338e664d3..dacaabbe4 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -18,11 +18,10 @@ import ( "strings" "testing" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/watch" - api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2" "github.com/coreos/etcd-operator/pkg/cluster" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" ) func TestHandleClusterEventUpdateFailedCluster(t *testing.T) { @@ -73,3 +72,66 @@ func TestHandleClusterEventDeleteFailedCluster(t *testing.T) { t.Errorf("failed cluster not cleaned up after delete event, cluster struct: %v", c.clusters[name]) } } + +func TestHandleClusterEventClusterwide(t *testing.T) { + c := New(Config{ClusterWide: true}) + + clus := &api.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Annotations: map[string]string{ + "etcd.database.coreos.com/scope": "clusterwide", + }, + }, + } + e := &Event{ + Type: watch.Modified, + Object: clus, + } + err := c.handleClusterEvent(e) + suffix := "isn't managed" + if strings.HasSuffix(err.Error(), suffix) { + t.Errorf("expect err='%s...', get=%v", suffix, err) + } +} + +func TestHandleClusterEventClusterwideIgnored(t *testing.T) { + c := New(Config{ClusterWide: true}) + + clus := &api.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + } + e := &Event{ + Type: watch.Modified, + Object: clus, + } + err := c.handleClusterEvent(e) + suffix := "isn't managed" + if !strings.HasSuffix(err.Error(), suffix) { + t.Errorf("expect err='%s...', get=%v", suffix, err) + } +} + +func TestHandleClusterEventNamespacedIgnored(t *testing.T) { + c := New(Config{}) + + clus := &api.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Annotations: map[string]string{ + "etcd.database.coreos.com/scope": "clusterwide", + }, + }, + } + e := &Event{ + Type: watch.Modified, + Object: clus, + } + err := c.handleClusterEvent(e) + suffix := "isn't managed" + if !strings.HasSuffix(err.Error(), suffix) { + t.Errorf("expect err='%s...', get=%v", suffix, err) + } +} diff --git a/pkg/controller/informer.go b/pkg/controller/informer.go index 72cf4cc97..a31a3fc8d 100644 --- a/pkg/controller/informer.go +++ b/pkg/controller/informer.go @@ -20,8 +20,10 @@ import ( "time" api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2" + "github.com/coreos/etcd-operator/pkg/util/constants" "github.com/coreos/etcd-operator/pkg/util/probe" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" kwatch "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/tools/cache" @@ -52,10 +54,17 @@ func (c *Controller) Start() error { } func (c *Controller) run() { + var ns string + if c.Config.ClusterWide { + ns = metav1.NamespaceAll + } else { + ns = c.Config.Namespace + } + source := cache.NewListWatchFromClient( c.Config.EtcdCRCli.EtcdV1beta2().RESTClient(), api.EtcdClusterResourcePlural, - c.Config.Namespace, + ns, fields.Everything()) _, informer := cache.NewIndexerInformer(source, &api.EtcdCluster{}, 0, cache.ResourceEventHandlerFuncs{ @@ -131,3 +140,16 @@ func (c *Controller) syncEtcdClus(clus *api.EtcdCluster) { } pt.stop() } + +func (c *Controller) managed(clus *api.EtcdCluster) bool { + if v, ok := clus.Annotations[constants.AnnotationScope]; ok { + if c.Config.ClusterWide { + return v == constants.AnnotationClusterWide + } + } else { + if !c.Config.ClusterWide { + return true + } + } + return false +} diff --git a/pkg/util/constants/constants.go b/pkg/util/constants/constants.go index e992dba39..a67a035f1 100644 --- a/pkg/util/constants/constants.go +++ b/pkg/util/constants/constants.go @@ -30,4 +30,7 @@ const ( EnvOperatorPodName = "MY_POD_NAME" EnvOperatorPodNamespace = "MY_POD_NAMESPACE" + + AnnotationScope = "etcd.database.coreos.com/scope" + AnnotationClusterWide = "clusterwide" ) From 06c9fd5323007edf7b8e7189a71aef0c24b85cf4 Mon Sep 17 00:00:00 2001 From: Guilhem Lettron Date: Sun, 4 Feb 2018 20:43:05 +0100 Subject: [PATCH 02/14] Do not report error when a cluster is ignored --- pkg/controller/controller.go | 18 +++++++++--------- pkg/controller/controller_test.go | 22 ++++++++-------------- pkg/controller/informer.go | 4 ++-- 3 files changed, 19 insertions(+), 25 deletions(-) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 17b7b2d7d..ec4cb27d3 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -62,32 +62,32 @@ func New(cfg Config) *Controller { } } -func (c *Controller) handleClusterEvent(event *Event) error { +func (c *Controller) handleClusterEvent(event *Event) (bool, error) { clus := event.Object if !c.managed(clus) { - return fmt.Errorf("cluster (%s) isn't managed", clus.Name) + return true, nil } if clus.Status.IsFailed() { clustersFailed.Inc() if event.Type == kwatch.Deleted { delete(c.clusters, clus.Name) - return nil + return false, nil } - return fmt.Errorf("ignore failed cluster (%s). Please delete its CR", clus.Name) + return false, fmt.Errorf("ignore failed cluster (%s). Please delete its CR", clus.Name) } clus.SetDefaults() if err := clus.Spec.Validate(); err != nil { - return fmt.Errorf("invalid cluster spec. please fix the following problem with the cluster spec: %v", err) + return false, fmt.Errorf("invalid cluster spec. please fix the following problem with the cluster spec: %v", err) } switch event.Type { case kwatch.Added: if _, ok := c.clusters[clus.Name]; ok { - return fmt.Errorf("unsafe state. cluster (%s) was created before but we received event (%s)", clus.Name, event.Type) + return false, fmt.Errorf("unsafe state. cluster (%s) was created before but we received event (%s)", clus.Name, event.Type) } nc := cluster.New(c.makeClusterConfig(), clus) @@ -99,21 +99,21 @@ func (c *Controller) handleClusterEvent(event *Event) error { case kwatch.Modified: if _, ok := c.clusters[clus.Name]; !ok { - return fmt.Errorf("unsafe state. cluster (%s) was never created but we received event (%s)", clus.Name, event.Type) + return false, fmt.Errorf("unsafe state. cluster (%s) was never created but we received event (%s)", clus.Name, event.Type) } c.clusters[clus.Name].Update(clus) clustersModified.Inc() case kwatch.Deleted: if _, ok := c.clusters[clus.Name]; !ok { - return fmt.Errorf("unsafe state. cluster (%s) was never created but we received event (%s)", clus.Name, event.Type) + return false, fmt.Errorf("unsafe state. cluster (%s) was never created but we received event (%s)", clus.Name, event.Type) } c.clusters[clus.Name].Delete() delete(c.clusters, clus.Name) clustersDeleted.Inc() clustersTotal.Dec() } - return nil + return false, nil } func (c *Controller) makeClusterConfig() cluster.Config { diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index dacaabbe4..fde2100a3 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -39,7 +39,7 @@ func TestHandleClusterEventUpdateFailedCluster(t *testing.T) { Type: watch.Modified, Object: clus, } - err := c.handleClusterEvent(e) + _, err := c.handleClusterEvent(e) prefix := "ignore failed cluster" if !strings.HasPrefix(err.Error(), prefix) { t.Errorf("expect err='%s...', get=%v", prefix, err) @@ -64,7 +64,7 @@ func TestHandleClusterEventDeleteFailedCluster(t *testing.T) { c.clusters[name] = &cluster.Cluster{} - if err := c.handleClusterEvent(e); err != nil { + if _, err := c.handleClusterEvent(e); err != nil { t.Fatal(err) } @@ -88,10 +88,8 @@ func TestHandleClusterEventClusterwide(t *testing.T) { Type: watch.Modified, Object: clus, } - err := c.handleClusterEvent(e) - suffix := "isn't managed" - if strings.HasSuffix(err.Error(), suffix) { - t.Errorf("expect err='%s...', get=%v", suffix, err) + if ignored, _ := c.handleClusterEvent(e); ignored { + t.Errorf("cluster shouldn't be ignored") } } @@ -107,10 +105,8 @@ func TestHandleClusterEventClusterwideIgnored(t *testing.T) { Type: watch.Modified, Object: clus, } - err := c.handleClusterEvent(e) - suffix := "isn't managed" - if !strings.HasSuffix(err.Error(), suffix) { - t.Errorf("expect err='%s...', get=%v", suffix, err) + if ignored, _ := c.handleClusterEvent(e); !ignored { + t.Errorf("cluster should be ignored") } } @@ -129,9 +125,7 @@ func TestHandleClusterEventNamespacedIgnored(t *testing.T) { Type: watch.Modified, Object: clus, } - err := c.handleClusterEvent(e) - suffix := "isn't managed" - if !strings.HasSuffix(err.Error(), suffix) { - t.Errorf("expect err='%s...', get=%v", suffix, err) + if ignored, _ := c.handleClusterEvent(e); !ignored { + t.Errorf("cluster should be ignored") } } diff --git a/pkg/controller/informer.go b/pkg/controller/informer.go index a31a3fc8d..0d14b78f0 100644 --- a/pkg/controller/informer.go +++ b/pkg/controller/informer.go @@ -114,7 +114,7 @@ func (c *Controller) onDeleteEtcdClus(obj interface{}) { } pt.start() - err := c.handleClusterEvent(ev) + _, err := c.handleClusterEvent(ev) if err != nil { c.logger.Warningf("fail to handle event: %v", err) } @@ -134,7 +134,7 @@ func (c *Controller) syncEtcdClus(clus *api.EtcdCluster) { } pt.start() - err := c.handleClusterEvent(ev) + _, err := c.handleClusterEvent(ev) if err != nil { c.logger.Warningf("fail to handle event: %v", err) } From ba95541e4b83402396a704f92fa1c005dd762701 Mon Sep 17 00:00:00 2001 From: Guilhem Lettron Date: Mon, 5 Feb 2018 16:17:27 +0100 Subject: [PATCH 03/14] Add clusterwide cluster example --- doc/user/clusterwide.md | 16 +--------------- example/example-etcd-cluster-clusterwide.yaml | 11 +++++++++++ 2 files changed, 12 insertions(+), 15 deletions(-) create mode 100644 example/example-etcd-cluster-clusterwide.yaml diff --git a/doc/user/clusterwide.md b/doc/user/clusterwide.md index a4ddaa777..612f5594e 100644 --- a/doc/user/clusterwide.md +++ b/doc/user/clusterwide.md @@ -9,18 +9,4 @@ etcd operator have to run with `-cluster-wide` arg option. More information in [install guide](doc/user/install_guide.md). -## Special annotation - -To declare an etcd cluster as "clusterwide", you have to add special annotation `etcd.database.coreos.com/scope` with value `clusterwide`. - -```yaml -apiVersion: "etcd.database.coreos.com/v1beta2" -kind: "EtcdCluster" -metadata: - name: "example-etcd-cluster" - annotations: - etcd.database.coreos.com/scope: clusterwide -spec: - size: 3 - version: "3.2.13" -``` +## [Cluster resource example](example/example-etcd-cluster-clusterwide.yaml) diff --git a/example/example-etcd-cluster-clusterwide.yaml b/example/example-etcd-cluster-clusterwide.yaml new file mode 100644 index 000000000..3d008b820 --- /dev/null +++ b/example/example-etcd-cluster-clusterwide.yaml @@ -0,0 +1,11 @@ +apiVersion: "etcd.database.coreos.com/v1beta2" +kind: "EtcdCluster" +metadata: + name: "example-etcd-cluster" + # Adding this annotation make this cluster managed by clusterwide operators + # namespaced operators ignore it + annotations: + etcd.database.coreos.com/scope: clusterwide +spec: + size: 3 + version: "3.2.13" From 38f7637f68348740d67c54881b63f10472e08ac1 Mon Sep 17 00:00:00 2001 From: Guilhem Lettron Date: Mon, 5 Feb 2018 16:28:03 +0100 Subject: [PATCH 04/14] Use named return values --- pkg/controller/controller.go | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index ec4cb27d3..43d81b640 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -62,32 +62,37 @@ func New(cfg Config) *Controller { } } -func (c *Controller) handleClusterEvent(event *Event) (bool, error) { +func (c *Controller) handleClusterEvent(event *Event) (ignored bool, err error) { clus := event.Object if !c.managed(clus) { - return true, nil + ignored = true + return } + ignored = false if clus.Status.IsFailed() { clustersFailed.Inc() if event.Type == kwatch.Deleted { delete(c.clusters, clus.Name) - return false, nil + return } - return false, fmt.Errorf("ignore failed cluster (%s). Please delete its CR", clus.Name) + err = fmt.Errorf("ignore failed cluster (%s). Please delete its CR", clus.Name) + return } clus.SetDefaults() - if err := clus.Spec.Validate(); err != nil { - return false, fmt.Errorf("invalid cluster spec. please fix the following problem with the cluster spec: %v", err) + if e := clus.Spec.Validate(); e != nil { + err = fmt.Errorf("invalid cluster spec. please fix the following problem with the cluster spec: %v", err) + return } switch event.Type { case kwatch.Added: if _, ok := c.clusters[clus.Name]; ok { - return false, fmt.Errorf("unsafe state. cluster (%s) was created before but we received event (%s)", clus.Name, event.Type) + err = fmt.Errorf("unsafe state. cluster (%s) was created before but we received event (%s)", clus.Name, event.Type) + return } nc := cluster.New(c.makeClusterConfig(), clus) @@ -99,21 +104,23 @@ func (c *Controller) handleClusterEvent(event *Event) (bool, error) { case kwatch.Modified: if _, ok := c.clusters[clus.Name]; !ok { - return false, fmt.Errorf("unsafe state. cluster (%s) was never created but we received event (%s)", clus.Name, event.Type) + err = fmt.Errorf("unsafe state. cluster (%s) was never created but we received event (%s)", clus.Name, event.Type) + return } c.clusters[clus.Name].Update(clus) clustersModified.Inc() case kwatch.Deleted: if _, ok := c.clusters[clus.Name]; !ok { - return false, fmt.Errorf("unsafe state. cluster (%s) was never created but we received event (%s)", clus.Name, event.Type) + err = fmt.Errorf("unsafe state. cluster (%s) was never created but we received event (%s)", clus.Name, event.Type) + return } c.clusters[clus.Name].Delete() delete(c.clusters, clus.Name) clustersDeleted.Inc() clustersTotal.Dec() } - return false, nil + return } func (c *Controller) makeClusterConfig() cluster.Config { From 1bc23a1d8ee48d791e7fcf15265fccfd173f11f2 Mon Sep 17 00:00:00 2001 From: Guilhem Lettron Date: Mon, 12 Feb 2018 18:40:15 +0100 Subject: [PATCH 05/14] Mix examples --- example/example-etcd-cluster-clusterwide.yaml | 11 ----------- example/example-etcd-cluster.yaml | 4 ++++ 2 files changed, 4 insertions(+), 11 deletions(-) delete mode 100644 example/example-etcd-cluster-clusterwide.yaml diff --git a/example/example-etcd-cluster-clusterwide.yaml b/example/example-etcd-cluster-clusterwide.yaml deleted file mode 100644 index 3d008b820..000000000 --- a/example/example-etcd-cluster-clusterwide.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: "etcd.database.coreos.com/v1beta2" -kind: "EtcdCluster" -metadata: - name: "example-etcd-cluster" - # Adding this annotation make this cluster managed by clusterwide operators - # namespaced operators ignore it - annotations: - etcd.database.coreos.com/scope: clusterwide -spec: - size: 3 - version: "3.2.13" diff --git a/example/example-etcd-cluster.yaml b/example/example-etcd-cluster.yaml index e75d261d8..8f5144122 100644 --- a/example/example-etcd-cluster.yaml +++ b/example/example-etcd-cluster.yaml @@ -2,6 +2,10 @@ apiVersion: "etcd.database.coreos.com/v1beta2" kind: "EtcdCluster" metadata: name: "example-etcd-cluster" + ## Adding this annotation make this cluster managed by clusterwide operators + ## namespaced operators ignore it + # annotations: + # etcd.database.coreos.com/scope: clusterwide spec: size: 3 version: "3.2.13" From 605ef99a49d0bb4d215312b34e6777db53f59dcc Mon Sep 17 00:00:00 2001 From: Guilhem Lettron Date: Mon, 12 Feb 2018 18:41:08 +0100 Subject: [PATCH 06/14] Explicit return and doc for handleClusterEvent --- pkg/controller/controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 43d81b640..4359dac73 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -62,12 +62,12 @@ func New(cfg Config) *Controller { } } +// handleClusterEvent returns ignored true if cluster isn't managed by this instance. func (c *Controller) handleClusterEvent(event *Event) (ignored bool, err error) { clus := event.Object if !c.managed(clus) { - ignored = true - return + return true, nil } ignored = false From 8ebe2f12eae68d53b31063090ca3581c44f3f6e9 Mon Sep 17 00:00:00 2001 From: Guilhem Lettron Date: Mon, 12 Feb 2018 18:42:02 +0100 Subject: [PATCH 07/14] Move constants to k8sutil --- pkg/controller/informer.go | 6 +++--- pkg/util/k8sutil/k8sutil.go | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/controller/informer.go b/pkg/controller/informer.go index 0d14b78f0..11424a53b 100644 --- a/pkg/controller/informer.go +++ b/pkg/controller/informer.go @@ -20,7 +20,7 @@ import ( "time" api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2" - "github.com/coreos/etcd-operator/pkg/util/constants" + "github.com/coreos/etcd-operator/pkg/util/k8sutil" "github.com/coreos/etcd-operator/pkg/util/probe" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -142,9 +142,9 @@ func (c *Controller) syncEtcdClus(clus *api.EtcdCluster) { } func (c *Controller) managed(clus *api.EtcdCluster) bool { - if v, ok := clus.Annotations[constants.AnnotationScope]; ok { + if v, ok := clus.Annotations[k8sutil.AnnotationScope]; ok { if c.Config.ClusterWide { - return v == constants.AnnotationClusterWide + return v == k8sutil.AnnotationClusterWide } } else { if !c.Config.ClusterWide { diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 0f7461ea9..01cb18739 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -62,6 +62,11 @@ const ( maxNameLength = 63 - randomSuffixLength - 1 defaultKubeAPIRequestTimeout = 30 * time.Second + + // AnnotationScope annotation name for defining instance scope. Used for specifing cluster wide clusters. + AnnotationScope = "etcd.database.coreos.com/scope" + //AnnotationClusterWide annotation value for cluster wide clusters. + AnnotationClusterWide = "clusterwide" ) const TolerateUnreadyEndpointsAnnotation = "service.alpha.kubernetes.io/tolerate-unready-endpoints" From 2b2920b9b9a0d6649dba168c9bf998691b3d853c Mon Sep 17 00:00:00 2001 From: Guilhem Lettron Date: Tue, 13 Feb 2018 18:15:38 +0100 Subject: [PATCH 08/14] Remove unused constants --- pkg/util/constants/constants.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/util/constants/constants.go b/pkg/util/constants/constants.go index a67a035f1..e992dba39 100644 --- a/pkg/util/constants/constants.go +++ b/pkg/util/constants/constants.go @@ -30,7 +30,4 @@ const ( EnvOperatorPodName = "MY_POD_NAME" EnvOperatorPodNamespace = "MY_POD_NAMESPACE" - - AnnotationScope = "etcd.database.coreos.com/scope" - AnnotationClusterWide = "clusterwide" ) From 18c07316420496acd50fe4e9434d2d182fb48ed4 Mon Sep 17 00:00:00 2001 From: Guilhem Lettron Date: Tue, 13 Feb 2018 18:16:05 +0100 Subject: [PATCH 09/14] Remove useless limitation section --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 0a1b66ac4..84cf9e5c3 100644 --- a/README.md +++ b/README.md @@ -282,7 +282,4 @@ Follow the [etcd restore operator walkthrough](./doc/user/walkthrough/restore-op See [instructions on clusterwide feature](doc/user/clusterwide.md). -### Limitations - - [k8s-home]: http://kubernetes.io From e03e637ba3312cd475f42ab09a6465417a42c1d1 Mon Sep 17 00:00:00 2001 From: Guilhem Lettron Date: Tue, 13 Feb 2018 18:16:48 +0100 Subject: [PATCH 10/14] Command help rephrasing --- cmd/operator/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 7e2cb2fa6..340793a45 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -68,7 +68,7 @@ func init() { flag.BoolVar(&printVersion, "version", false, "Show version and quit") flag.BoolVar(&createCRD, "create-crd", true, "The operator will not create the EtcdCluster CRD when this flag is set to false.") flag.DurationVar(&gcInterval, "gc-interval", 10*time.Minute, "GC interval") - flag.BoolVar(&clusterWide, "cluster-wide", false, "The operator will watch clusters in all namespaces") + flag.BoolVar(&clusterWide, "cluster-wide", false, "Enable operator to watch clusters in all namespaces") flag.Parse() } From a4f4b063a6a61343d1b57b84f57184c1c681f2a5 Mon Sep 17 00:00:00 2001 From: Guilhem Lettron Date: Tue, 13 Feb 2018 18:30:57 +0100 Subject: [PATCH 11/14] Clusterwide example --- doc/user/clusterwide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/user/clusterwide.md b/doc/user/clusterwide.md index 612f5594e..52749e145 100644 --- a/doc/user/clusterwide.md +++ b/doc/user/clusterwide.md @@ -9,4 +9,4 @@ etcd operator have to run with `-cluster-wide` arg option. More information in [install guide](doc/user/install_guide.md). -## [Cluster resource example](example/example-etcd-cluster-clusterwide.yaml) +See the example in [example/example-etcd-cluster.yaml](example/example-etcd-cluster.yaml) From fd0e59bb52ebacd5472cadf60860dcd11ce24780 Mon Sep 17 00:00:00 2001 From: Guilhem Lettron Date: Tue, 13 Feb 2018 18:33:39 +0100 Subject: [PATCH 12/14] handleClusterEvent help --- pkg/controller/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 4359dac73..c11049db0 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -62,8 +62,8 @@ func New(cfg Config) *Controller { } } -// handleClusterEvent returns ignored true if cluster isn't managed by this instance. func (c *Controller) handleClusterEvent(event *Event) (ignored bool, err error) { +// handleClusterEvent returns true if cluster is ignored (not managed) by this instance. clus := event.Object if !c.managed(clus) { From cc6cd9ef879bc67587f622e4e75b5ebc7880898e Mon Sep 17 00:00:00 2001 From: Guilhem Lettron Date: Tue, 13 Feb 2018 18:34:04 +0100 Subject: [PATCH 13/14] Explicit return values for handleClusterEvent --- pkg/controller/controller.go | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index c11049db0..499fcc070 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -62,37 +62,33 @@ func New(cfg Config) *Controller { } } -func (c *Controller) handleClusterEvent(event *Event) (ignored bool, err error) { // handleClusterEvent returns true if cluster is ignored (not managed) by this instance. +func (c *Controller) handleClusterEvent(event *Event) (bool, error) { clus := event.Object if !c.managed(clus) { return true, nil } - ignored = false if clus.Status.IsFailed() { clustersFailed.Inc() if event.Type == kwatch.Deleted { delete(c.clusters, clus.Name) - return + return false, nil } - err = fmt.Errorf("ignore failed cluster (%s). Please delete its CR", clus.Name) - return + return false, fmt.Errorf("ignore failed cluster (%s). Please delete its CR", clus.Name) } clus.SetDefaults() - if e := clus.Spec.Validate(); e != nil { - err = fmt.Errorf("invalid cluster spec. please fix the following problem with the cluster spec: %v", err) - return + if err := clus.Spec.Validate(); err != nil { + return false, fmt.Errorf("invalid cluster spec. please fix the following problem with the cluster spec: %v", err) } switch event.Type { case kwatch.Added: if _, ok := c.clusters[clus.Name]; ok { - err = fmt.Errorf("unsafe state. cluster (%s) was created before but we received event (%s)", clus.Name, event.Type) - return + return false, fmt.Errorf("unsafe state. cluster (%s) was created before but we received event (%s)", clus.Name, event.Type) } nc := cluster.New(c.makeClusterConfig(), clus) @@ -104,23 +100,21 @@ func (c *Controller) handleClusterEvent(event *Event) (ignored bool, err error) case kwatch.Modified: if _, ok := c.clusters[clus.Name]; !ok { - err = fmt.Errorf("unsafe state. cluster (%s) was never created but we received event (%s)", clus.Name, event.Type) - return + return false, fmt.Errorf("unsafe state. cluster (%s) was never created but we received event (%s)", clus.Name, event.Type) } c.clusters[clus.Name].Update(clus) clustersModified.Inc() case kwatch.Deleted: if _, ok := c.clusters[clus.Name]; !ok { - err = fmt.Errorf("unsafe state. cluster (%s) was never created but we received event (%s)", clus.Name, event.Type) - return + return false, fmt.Errorf("unsafe state. cluster (%s) was never created but we received event (%s)", clus.Name, event.Type) } c.clusters[clus.Name].Delete() delete(c.clusters, clus.Name) clustersDeleted.Inc() clustersTotal.Dec() } - return + return false, nil } func (c *Controller) makeClusterConfig() cluster.Config { From 11ce31f03ab8c63a51dfe84ec9807d846f1bf6a7 Mon Sep 17 00:00:00 2001 From: Guilhem Lettron Date: Tue, 13 Feb 2018 18:50:10 +0100 Subject: [PATCH 14/14] Fix relative link to example file --- doc/user/clusterwide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/user/clusterwide.md b/doc/user/clusterwide.md index 52749e145..e43761e42 100644 --- a/doc/user/clusterwide.md +++ b/doc/user/clusterwide.md @@ -9,4 +9,4 @@ etcd operator have to run with `-cluster-wide` arg option. More information in [install guide](doc/user/install_guide.md). -See the example in [example/example-etcd-cluster.yaml](example/example-etcd-cluster.yaml) +See the example in [example/example-etcd-cluster.yaml](../../example/example-etcd-cluster.yaml)