Allow pausing clusters

docs/cassandra.rst

@@ -175,6 +175,8 @@ For example, you could use the `Datastax Python driver <http://datastax.github.i
    The IP address to which the driver makes the initial connection
    depends on the DNS server and operating system configuration.
 
+.. include:: paused-clusters.rst
+
 .. include:: supplementary-resources.rst
 
 The Life Cycle of a Navigator Cassandra Cluster

docs/elasticsearch.rst

@@ -23,6 +23,8 @@ and in Navigator, these groups of nodes are called ``nodepools``.
 
 .. include:: configure-scheduler.rst
 
+.. include:: paused-clusters.rst
+
 .. include:: managing-compute-resources.rst
 
 .. include:: supplementary-resources.rst

docs/paused-clusters.rst

@@ -0,0 +1,5 @@
+Pausing Clusters
+----------------
+
+A cluster can be paused by setting ``spec.paused`` to ``true``.
+When this is set, Navigator will not perform any actions on the cluster, allowing for manual intervention.

docs/quick-start/cassandra-cluster.yaml

@@ -3,6 +3,9 @@ kind: "CassandraCluster"
 metadata:
   name: "demo"
 spec:
+  #  if set, navigator will not perform any actions on the cluster
+  # paused: true
+
   version: "3.11.1"
   nodePools:
   - name: "region-1-zone-a"

docs/quick-start/es-cluster-demo.yaml

@@ -3,6 +3,9 @@ kind: ElasticsearchCluster
 metadata:
   name: demo
 spec:
+  #  if set, navigator will not perform any actions on the cluster
+  # paused: true
+
   ## Omitting the minimumMasters fields will cause navigator to automatically
   ## determine a quorum of masters to use.
   # minimumMasters: 2

pkg/apis/navigator/types.go

@@ -46,6 +46,7 @@ type CassandraClusterNodePool struct {
 }
 
 type CassandraClusterStatus struct {
+	NavigatorClusterStatus
 	NodePools map[string]CassandraClusterNodePoolStatus
 }
 
@@ -79,6 +80,7 @@ type ElasticsearchCluster struct {
 }
 
 type ElasticsearchClusterStatus struct {
+	NavigatorClusterStatus
 	NodePools map[string]ElasticsearchClusterNodePoolStatus
 	Health    *ElasticsearchClusterHealth
 }
@@ -150,12 +152,43 @@ type NavigatorClusterConfig struct {
 	PilotImage ImageSpec
 
 	SecurityContext NavigatorSecurityContext
+
+	Paused bool
+}
+
+type NavigatorClusterStatus struct {
+	Conditions []ClusterCondition
 }
 
 type NavigatorSecurityContext struct {
 	RunAsUser *int64
 }
 
+type ClusterConditionType string
+
+const (
+	ClusterConditionAvailable ClusterConditionType = "Available"
+
+	ClusterConditionProgressing ClusterConditionType = "Progressing"
+)
+
+type ClusterCondition struct {
+	// Type of cluster condition.
+	Type ClusterConditionType
+
+	// Status of the condition, one of True, False, Unknown.
+	Status ConditionStatus
+
+	// Last time the condition transitioned from one status to another.
+	LastTransitionTime metav1.Time
+
+	// The reason for the condition's last transition.
+	Reason string
+
+	// A human readable message indicating details about the transition.
+	Message string
+}
+
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 

pkg/apis/navigator/v1alpha1/helpers.go

@@ -50,3 +50,42 @@ func (p *Pilot) UpdateStatusCondition(conditionType PilotConditionType, status C
 		}
 	}
 }
+
+func (s *NavigatorClusterStatus) GetStatusCondition(conditionType ClusterConditionType) *ClusterCondition {
+	for i := range s.Conditions {
+		c := s.Conditions[i]
+		if c.Type == conditionType {
+			return &c
+		}
+	}
+	return nil
+}
+
+func (c *NavigatorClusterStatus) UpdateStatusCondition(conditionType ClusterConditionType, status ConditionStatus, reason, message string) {
+	newCondition := ClusterCondition{
+		Type:    conditionType,
+		Status:  status,
+		Reason:  reason,
+		Message: message,
+	}
+
+	t := time.Now()
+
+	if len(c.Conditions) == 0 {
+		newCondition.LastTransitionTime = metav1.NewTime(t)
+		c.Conditions = []ClusterCondition{newCondition}
+	} else {
+		for i, cond := range c.Conditions {
+			if cond.Type == conditionType {
+				if cond.Status != newCondition.Status {
+					newCondition.LastTransitionTime = metav1.NewTime(t)
+				} else {
+					newCondition.LastTransitionTime = cond.LastTransitionTime
+				}
+
+				c.Conditions[i] = newCondition
+				break
+			}
+		}
+	}
+}

pkg/apis/navigator/v1alpha1/types.go

@@ -106,6 +106,8 @@ type CassandraClusterNodePool struct {
 }
 
 type CassandraClusterStatus struct {
+	NavigatorClusterStatus `json:",inline"`
+
 	// The status of each node pool in the CassandraCluster.
 	// This will be periodically updated by Navigator.
 	NodePools map[string]CassandraClusterNodePoolStatus `json:"nodePools,omitempty"`
@@ -143,6 +145,8 @@ type ElasticsearchCluster struct {
 // ElasticsearchClusterStatus specifies the overall status of an
 // ElasticsearchCluster.
 type ElasticsearchClusterStatus struct {
+	NavigatorClusterStatus `json:",inline"`
+
 	// The status of each node pool in the ElasticsearchCluster.
 	// This will be periodically updated by Navigator.
 	NodePools map[string]ElasticsearchClusterNodePoolStatus `json:"nodePools,omitempty"`
@@ -297,6 +301,16 @@ type NavigatorClusterConfig struct {
 
 	// Security related options to be applied to the cluster pods.
 	SecurityContext NavigatorSecurityContext `json:"securityContext"`
+
+	// If set to true, no actions will take place on this cluster.
+	Paused bool `json:"paused,omitempty"`
+}
+
+type NavigatorClusterStatus struct {
+	// Represents the latest available observations of a cluster's current state.
+	// +patchMergeKey=type
+	// +patchStrategy=merge
+	Conditions []ClusterCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"`
 }
 
 type NavigatorSecurityContext struct {
@@ -308,6 +322,39 @@ type NavigatorSecurityContext struct {
 	RunAsUser *int64 `json:"runAsUser,omitempty"`
 }
 
+type ClusterConditionType string
+
+const (
+	ClusterConditionAvailable ClusterConditionType = "Available"
+
+	ClusterConditionProgressing ClusterConditionType = "Progressing"
+)
+
+const (
+	// PausedClusterReason is added in a cluster when it is paused. Lack of progress shouldn't be
+	// estimated once a cluster is paused.
+	PausedClusterReason = "ClusterPaused"
+	// ResumedClusterReason is added in a cluster when it is resumed.
+	ResumedClusterReason = "ClusterResumed"
+)
+
+type ClusterCondition struct {
+	// Type of cluster condition.
+	Type ClusterConditionType `json:"type"`
+
+	// Status of the condition, one of True, False, Unknown.
+	Status ConditionStatus `json:"status"`
+
+	// Last time the condition transitioned from one status to another.
+	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
+
+	// The reason for the condition's last transition.
+	Reason string `json:"reason,omitempty"`
+
+	// A human readable message indicating details about the transition.
+	Message string `json:"message,omitempty"`
+}
+
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 

pkg/apis/navigator/v1alpha1/zz_generated.conversion.go

@@ -52,6 +52,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
 		Convert_navigator_CassandraClusterStatus_To_v1alpha1_CassandraClusterStatus,
 		Convert_v1alpha1_CassandraPilotStatus_To_navigator_CassandraPilotStatus,
 		Convert_navigator_CassandraPilotStatus_To_v1alpha1_CassandraPilotStatus,
+		Convert_v1alpha1_ClusterCondition_To_navigator_ClusterCondition,
+		Convert_navigator_ClusterCondition_To_v1alpha1_ClusterCondition,
 		Convert_v1alpha1_ElasticsearchCluster_To_navigator_ElasticsearchCluster,
 		Convert_navigator_ElasticsearchCluster_To_v1alpha1_ElasticsearchCluster,
 		Convert_v1alpha1_ElasticsearchClusterList_To_navigator_ElasticsearchClusterList,
@@ -70,6 +72,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
 		Convert_navigator_ImageSpec_To_v1alpha1_ImageSpec,
 		Convert_v1alpha1_NavigatorClusterConfig_To_navigator_NavigatorClusterConfig,
 		Convert_navigator_NavigatorClusterConfig_To_v1alpha1_NavigatorClusterConfig,
+		Convert_v1alpha1_NavigatorClusterStatus_To_navigator_NavigatorClusterStatus,
+		Convert_navigator_NavigatorClusterStatus_To_v1alpha1_NavigatorClusterStatus,
 		Convert_v1alpha1_NavigatorSecurityContext_To_navigator_NavigatorSecurityContext,
 		Convert_navigator_NavigatorSecurityContext_To_v1alpha1_NavigatorSecurityContext,
 		Convert_v1alpha1_PersistenceConfig_To_navigator_PersistenceConfig,
@@ -246,6 +250,9 @@ func Convert_navigator_CassandraClusterSpec_To_v1alpha1_CassandraClusterSpec(in
 }
 
 func autoConvert_v1alpha1_CassandraClusterStatus_To_navigator_CassandraClusterStatus(in *CassandraClusterStatus, out *navigator.CassandraClusterStatus, s conversion.Scope) error {
+	if err := Convert_v1alpha1_NavigatorClusterStatus_To_navigator_NavigatorClusterStatus(&in.NavigatorClusterStatus, &out.NavigatorClusterStatus, s); err != nil {
+		return err
+	}
 	out.NodePools = *(*map[string]navigator.CassandraClusterNodePoolStatus)(unsafe.Pointer(&in.NodePools))
 	return nil
 }
@@ -256,6 +263,9 @@ func Convert_v1alpha1_CassandraClusterStatus_To_navigator_CassandraClusterStatus
 }
 
 func autoConvert_navigator_CassandraClusterStatus_To_v1alpha1_CassandraClusterStatus(in *navigator.CassandraClusterStatus, out *CassandraClusterStatus, s conversion.Scope) error {
+	if err := Convert_navigator_NavigatorClusterStatus_To_v1alpha1_NavigatorClusterStatus(&in.NavigatorClusterStatus, &out.NavigatorClusterStatus, s); err != nil {
+		return err
+	}
 	out.NodePools = *(*map[string]CassandraClusterNodePoolStatus)(unsafe.Pointer(&in.NodePools))
 	return nil
 }
@@ -285,6 +295,34 @@ func Convert_navigator_CassandraPilotStatus_To_v1alpha1_CassandraPilotStatus(in
 	return autoConvert_navigator_CassandraPilotStatus_To_v1alpha1_CassandraPilotStatus(in, out, s)
 }
 
+func autoConvert_v1alpha1_ClusterCondition_To_navigator_ClusterCondition(in *ClusterCondition, out *navigator.ClusterCondition, s conversion.Scope) error {
+	out.Type = navigator.ClusterConditionType(in.Type)
+	out.Status = navigator.ConditionStatus(in.Status)
+	out.LastTransitionTime = in.LastTransitionTime
+	out.Reason = in.Reason
+	out.Message = in.Message
+	return nil
+}
+
+// Convert_v1alpha1_ClusterCondition_To_navigator_ClusterCondition is an autogenerated conversion function.
+func Convert_v1alpha1_ClusterCondition_To_navigator_ClusterCondition(in *ClusterCondition, out *navigator.ClusterCondition, s conversion.Scope) error {
+	return autoConvert_v1alpha1_ClusterCondition_To_navigator_ClusterCondition(in, out, s)
+}
+
+func autoConvert_navigator_ClusterCondition_To_v1alpha1_ClusterCondition(in *navigator.ClusterCondition, out *ClusterCondition, s conversion.Scope) error {
+	out.Type = ClusterConditionType(in.Type)
+	out.Status = ConditionStatus(in.Status)
+	out.LastTransitionTime = in.LastTransitionTime
+	out.Reason = in.Reason
+	out.Message = in.Message
+	return nil
+}
+
+// Convert_navigator_ClusterCondition_To_v1alpha1_ClusterCondition is an autogenerated conversion function.
+func Convert_navigator_ClusterCondition_To_v1alpha1_ClusterCondition(in *navigator.ClusterCondition, out *ClusterCondition, s conversion.Scope) error {
+	return autoConvert_navigator_ClusterCondition_To_v1alpha1_ClusterCondition(in, out, s)
+}
+
 func autoConvert_v1alpha1_ElasticsearchCluster_To_navigator_ElasticsearchCluster(in *ElasticsearchCluster, out *navigator.ElasticsearchCluster, s conversion.Scope) error {
 	out.ObjectMeta = in.ObjectMeta
 	if err := Convert_v1alpha1_ElasticsearchClusterSpec_To_navigator_ElasticsearchClusterSpec(&in.Spec, &out.Spec, s); err != nil {
@@ -446,6 +484,9 @@ func Convert_navigator_ElasticsearchClusterSpec_To_v1alpha1_ElasticsearchCluster
 }
 
 func autoConvert_v1alpha1_ElasticsearchClusterStatus_To_navigator_ElasticsearchClusterStatus(in *ElasticsearchClusterStatus, out *navigator.ElasticsearchClusterStatus, s conversion.Scope) error {
+	if err := Convert_v1alpha1_NavigatorClusterStatus_To_navigator_NavigatorClusterStatus(&in.NavigatorClusterStatus, &out.NavigatorClusterStatus, s); err != nil {
+		return err
+	}
 	out.NodePools = *(*map[string]navigator.ElasticsearchClusterNodePoolStatus)(unsafe.Pointer(&in.NodePools))
 	out.Health = (*navigator.ElasticsearchClusterHealth)(unsafe.Pointer(in.Health))
 	return nil
@@ -457,6 +498,9 @@ func Convert_v1alpha1_ElasticsearchClusterStatus_To_navigator_ElasticsearchClust
 }
 
 func autoConvert_navigator_ElasticsearchClusterStatus_To_v1alpha1_ElasticsearchClusterStatus(in *navigator.ElasticsearchClusterStatus, out *ElasticsearchClusterStatus, s conversion.Scope) error {
+	if err := Convert_navigator_NavigatorClusterStatus_To_v1alpha1_NavigatorClusterStatus(&in.NavigatorClusterStatus, &out.NavigatorClusterStatus, s); err != nil {
+		return err
+	}
 	out.NodePools = *(*map[string]ElasticsearchClusterNodePoolStatus)(unsafe.Pointer(&in.NodePools))
 	out.Health = (*ElasticsearchClusterHealth)(unsafe.Pointer(in.Health))
 	return nil
@@ -520,6 +564,7 @@ func autoConvert_v1alpha1_NavigatorClusterConfig_To_navigator_NavigatorClusterCo
 	if err := Convert_v1alpha1_NavigatorSecurityContext_To_navigator_NavigatorSecurityContext(&in.SecurityContext, &out.SecurityContext, s); err != nil {
 		return err
 	}
+	out.Paused = in.Paused
 	return nil
 }
 
@@ -535,6 +580,7 @@ func autoConvert_navigator_NavigatorClusterConfig_To_v1alpha1_NavigatorClusterCo
 	if err := Convert_navigator_NavigatorSecurityContext_To_v1alpha1_NavigatorSecurityContext(&in.SecurityContext, &out.SecurityContext, s); err != nil {
 		return err
 	}
+	out.Paused = in.Paused
 	return nil
 }
 
@@ -543,6 +589,26 @@ func Convert_navigator_NavigatorClusterConfig_To_v1alpha1_NavigatorClusterConfig
 	return autoConvert_navigator_NavigatorClusterConfig_To_v1alpha1_NavigatorClusterConfig(in, out, s)
 }
 
+func autoConvert_v1alpha1_NavigatorClusterStatus_To_navigator_NavigatorClusterStatus(in *NavigatorClusterStatus, out *navigator.NavigatorClusterStatus, s conversion.Scope) error {
+	out.Conditions = *(*[]navigator.ClusterCondition)(unsafe.Pointer(&in.Conditions))
+	return nil
+}
+
+// Convert_v1alpha1_NavigatorClusterStatus_To_navigator_NavigatorClusterStatus is an autogenerated conversion function.
+func Convert_v1alpha1_NavigatorClusterStatus_To_navigator_NavigatorClusterStatus(in *NavigatorClusterStatus, out *navigator.NavigatorClusterStatus, s conversion.Scope) error {
+	return autoConvert_v1alpha1_NavigatorClusterStatus_To_navigator_NavigatorClusterStatus(in, out, s)
+}
+
+func autoConvert_navigator_NavigatorClusterStatus_To_v1alpha1_NavigatorClusterStatus(in *navigator.NavigatorClusterStatus, out *NavigatorClusterStatus, s conversion.Scope) error {
+	out.Conditions = *(*[]ClusterCondition)(unsafe.Pointer(&in.Conditions))
+	return nil
+}
+
+// Convert_navigator_NavigatorClusterStatus_To_v1alpha1_NavigatorClusterStatus is an autogenerated conversion function.
+func Convert_navigator_NavigatorClusterStatus_To_v1alpha1_NavigatorClusterStatus(in *navigator.NavigatorClusterStatus, out *NavigatorClusterStatus, s conversion.Scope) error {
+	return autoConvert_navigator_NavigatorClusterStatus_To_v1alpha1_NavigatorClusterStatus(in, out, s)
+}
+
 func autoConvert_v1alpha1_NavigatorSecurityContext_To_navigator_NavigatorSecurityContext(in *NavigatorSecurityContext, out *navigator.NavigatorSecurityContext, s conversion.Scope) error {
 	out.RunAsUser = (*int64)(unsafe.Pointer(in.RunAsUser))
 	return nil