diff --git a/Makefile b/Makefile index 9e6cf72f..67049cc8 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ # Image URL to use all building/pushing image targets IMG ?= controller:202307101 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.25.0 +ENVTEST_K8S_VERSION = 1.22.1 # kind cluster name for e2e CLUSTER_NAME ?= kindcluster # kind version for e2e @@ -163,7 +163,7 @@ GINKGO ?= $(LOCALBIN)/ginkgo ## Tool Versions KUSTOMIZE_VERSION ?= v4.5.5 -CONTROLLER_TOOLS_VERSION ?= v0.12.0 +CONTROLLER_TOOLS_VERSION ?= v0.10.0 GINKGO_VERSION ?= 1.16.5 KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" diff --git a/apis/addtoscheme_apps_v1alpha1.go b/apis/addtoscheme_apps_v1alpha1.go new file mode 100644 index 00000000..654509eb --- /dev/null +++ b/apis/addtoscheme_apps_v1alpha1.go @@ -0,0 +1,26 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apis + +import ( + api "kusionstack.io/kafed/apis/apps/v1alpha1" +) + +func init() { + // Register the types with the Scheme so the components can map objects to GroupVersionKinds and back + AddToSchemes = append(AddToSchemes, api.SchemeBuilder.AddToScheme) +} diff --git a/apis/apps/v1alpha1/collaset_types.go b/apis/apps/v1alpha1/collaset_types.go new file mode 100644 index 00000000..586e3496 --- /dev/null +++ b/apis/apps/v1alpha1/collaset_types.go @@ -0,0 +1,279 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type CollaSetConditionType string + +const ( + CollaSetSyncPod CollaSetConditionType = "SyncPod" + CollaSetScale CollaSetConditionType = "Scale" + CollaSetUpdate CollaSetConditionType = "Update" +) + +// PersistentVolumeClaimRetentionPolicyType is a string enumeration of the policies that will determine +// which action will be applied on volumes from the VolumeClaimTemplates when the CollaSet is +// deleted or scaled down. +type PersistentVolumeClaimRetentionPolicyType string + +const ( + // RetainPersistentVolumeClaimRetentionPolicyType specifies that + // PersistentVolumeClaims associated with CollaSet VolumeClaimTemplates + // will not be deleted. + RetainPersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Retain" + // DeletePersistentVolumeClaimRetentionPolicyType is the default policy, which specifies that + // PersistentVolumeClaims associated with CollaSet VolumeClaimTemplates + // will be deleted in the scenario specified in PersistentVolumeClaimRetentionPolicy. + DeletePersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Delete" +) + +// PodUpdateStrategyType is a string enumeration type that enumerates +// all possible ways we can update a Pod when updating application +type PodUpdateStrategyType string + +const ( + // CollaSetRecreatePodUpdateStrategyType indicates that CollaSet will always update Pod by deleting and recreate it. + CollaSetRecreatePodUpdateStrategyType PodUpdateStrategyType = "ReCreate" + // CollaSetInPlaceIfPossiblePodUpdateStrategyType indicates thath CollaSet will try to update Pod by in-place update + // when it is possible. Recently, only Pod image can be updated in-place. Any other Pod spec change will make the + // policy fall back to CollaSetRecreatePodUpdateStrategyType. + CollaSetInPlaceIfPossiblePodUpdateStrategyType PodUpdateStrategyType = "InPlaceIfPossible" + // CollaSetInPlaceOnlyPodUpdateStrategyType indicates that CollaSet will always update Pod in-place, instead of + // recreating pod. It will encounter an error on original Kubernetes cluster. + CollaSetInPlaceOnlyPodUpdateStrategyType PodUpdateStrategyType = "InPlaceOnly" +) + +// CollaSetSpec defines the desired state of CollaSet +type CollaSetSpec struct { + // Indicates that the scaling and updating is paused and will not be processed by the + // CollaSet controller. + // +optional + Paused bool `json:"paused,omitempty"` + // Replicas is the desired number of replicas of the given Template. + // These are replicas in the sense that they are instantiations of the + // same Template, but individual replicas also have a consistent identity. + // If unspecified, defaults to 0. + // +optional + Replicas *int32 `json:"replicas,omitempty"` + + // Selector is a label query over pods that should match the replica count. + // It must match the pod template's labels. + Selector *metav1.LabelSelector `json:"selector,omitempty"` + + // Template is the object that describes the pod that will be created if + // insufficient replicas are detected. Each pod stamped out by the CollaSet + // will fulfill this Template, but have a unique identity from the rest + // of the CollaSet. + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + Template corev1.PodTemplateSpec `json:"template,omitempty"` + + // VolumeClaimTemplates is a list of claims that pods are allowed to reference. + // The StatefulSet controller is responsible for mapping network identities to + // claims in a way that maintains the identity of a pod. Every claim in + // this list must have at least one matching (by name) volumeMount in one + // container in the template. A claim in this list takes precedence over + // any volumes in the template, with the same name. + // +optional + VolumeClaimTemplates []corev1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"` + + // UpdateStrategy indicates the CollaSetUpdateStrategy that will be + // employed to update Pods in the CollaSet when a revision is made to + // Template. + UpdateStrategy UpdateStrategy `json:"updateStrategy,omitempty"` + + // ScaleStrategy indicates the strategy detail that will be used during pod scaling. + ScaleStrategy ScaleStrategy `json:"scaleStrategy,omitempty"` + + // Indicate the number of histories to be conserved + // If unspecified, defaults to 20 + // +optional + HistoryLimit int32 `json:"historyLimit,omitempty"` +} + +type ScaleStrategy struct { + // Context indicates the pool from which to allocate Pod instance ID. CollaSets are allowed to share the + // same Context. It is not allowed to change. + // Context defaults to be CollaSet's name. + Context string `json:"context,omitempty"` + // PodToExclude indicates the pods which will be orphaned by CollaSet. + PodToExclude []string `json:"podToExclude,omitempty"` + + // PodToInclude indicates the pods which will be adapted by CollaSet. + PodToInclude []string `json:"podToInclude,omitempty"` + + // PersistentVolumeClaimRetentionPolicy describes the lifecycle of PersistentVolumeClaim + // created from volumeClaimTemplates. By default, all persistent volume claims are created as needed and + // deleted after no pod is using them. This policy allows the lifecycle to be altered, for example + // by deleting persistent volume claims when their CollaSet is deleted, or when their pod is scaled down. + // +optional + PersistentVolumeClaimRetentionPolicy *PersistentVolumeClaimRetentionPolicy `json:"persistentVolumeClaimRetentionPolicy,omitempty"` +} + +type PersistentVolumeClaimRetentionPolicy struct { + // WhenDeleted specifies what happens to PVCs created from CollaSet + // VolumeClaimTemplates when the CollaSet is deleted. The default policy + // of `Delete` policy causes those PVCs to be deleted. + //`Retain` causes PVCs to not be affected by StatefulSet deletion. The + WhenDeleted PersistentVolumeClaimRetentionPolicyType `json:"whenDeleted,omitempty"` + // WhenScaled specifies what happens to PVCs created from StatefulSet + // VolumeClaimTemplates when the StatefulSet is scaled down. The default + // policy of `Retain` causes PVCs to not be affected by a scaledown. The + // `Delete` policy causes the associated PVCs for any excess pods above + // the replica count to be deleted. + WhenScaled PersistentVolumeClaimRetentionPolicyType `json:"whenScaled,omitempty"` +} + +type ByPartition struct { + // Partition controls the update progress by indicating how many pods should be updated. + // Defaults to nil (all pods will be updated) + Partition *int32 `json:"partition,omitempty"` +} + +type ByLabel struct { +} + +// RollingUpdateCollaSetStrategy is used to communicate parameter for rolling update. +type RollingUpdateCollaSetStrategy struct { + // ByPartition indicates the update progress is controlled by partition value. + ByPartition *ByPartition `json:"byPartition,omitempty"` + + // ByLabel indicates the update progress is controlled by attaching pod label. + ByLabel *ByLabel `json:"byLabel,omitempty"` +} + +type UpdateStrategy struct { + // RollingUpdate is used to communicate parameters when Type is RollingUpdateStatefulSetStrategyType. + RollingUpdate *RollingUpdateCollaSetStrategy `json:"rollingUpdate,omitempty"` + + // PodUpdatePolicy indicates the policy by to update pods. + // +optional + PodUpdatePolicy PodUpdateStrategyType `json:"podUpgradePolicy,omitempty"` +} + +// CollaSetStatus defines the observed state of CollaSet +type CollaSetStatus struct { + // ObservedGeneration is the most recent generation observed for this CollaSet. It corresponds to the + // CollaSet's generation, which is updated on mutation by the API Server. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // CurrentRevision, if not empty, indicates the version of the CollaSet. + CurrentRevision string `json:"currentRevision,omitempty"` + + // UpdatedRevision, if not empty, indicates the version of the CollaSet currently updated. + UpdatedRevision string `json:"updatedRevision,omitempty"` + + // Count of hash collisions for the DaemonSet. The DaemonSet controller + // uses this field as a collision avoidance mechanism when it needs to + // create the name for the newest ControllerRevision. + // +optional + CollisionCount *int32 `json:"collisionCount,omitempty"` + + // the number of scheduled replicas for the CollaSet. + // +optional + ScheduledReplicas int32 `json:"scheduledReplicas,omitempty"` + + // ReadyReplicas indicates the number of the pod with ready condition + // +optional + ReadyReplicas int32 `json:"readyReplicas,omitempty"` + + // The number of available replicas (ready for at least minReadySeconds) for this replica set. + // +optional + AvailableReplicas int32 `json:"availableReplicas,omitempty"` + + // Replicas is the most recently observed number of replicas. + Replicas int32 `json:"replicas,omitempty"` + + // The number of pods in updated version. + UpdatedReplicas int32 `json:"updatedReplicas,omitempty"` + + // OperatingReplicas indicates the number of pods during pod ops lifecycle and not finish update-phase. + OperatingReplicas int32 `json:"operatingReplicas,omitempty"` + + // UpdatedReadyReplicas indicates the number of the pod with updated revision and ready condition + // +optional + UpdatedReadyReplicas int32 `json:"updatedReadyReplicas,omitempty"` + + // UpdatedAvailableReplicas indicates the number of available updated revision replicas for this CollaSet. + // A pod is updated available means the pod is ready for updated revision and accessible + // +optional + UpdatedAvailableReplicas int32 `json:"updatedAvailableReplicas,omitempty"` + + // Represents the latest available observations of a CollaSet's current state. + // +optional + Conditions []CollaSetCondition `json:"conditions,omitempty"` +} + +type CollaSetCondition struct { + // Type of in place set condition. + Type CollaSetConditionType `json:"type,omitempty"` + + // Status of the condition, one of True, False, Unknown. + Status corev1.ConditionStatus `json:"status,omitempty"` + + // Last time the condition transitioned from one status to another. + LastTransitionTime metav1.Time `json:"last_transition_time,omitempty"` + + // The reason for the condition's last transition. + Reason string `json:"reason,omitempty"` + + // A human readable message indicating details about the transition. + Message string `json:"message,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// CollaSet is the Schema for the collasets API +// +k8s:openapi-gen=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:resource:shortName=cls +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="DESIRED",type="integer",JSONPath=".spec.replicas",description="The desired number of pods." +// +kubebuilder:printcolumn:name="CURRENT",type="integer",JSONPath=".status.replicas",description="The number of currently all pods." +// +kubebuilder:printcolumn:name="UPDATED",type="integer",JSONPath=".status.updatedReplicas",description="The number of pods updated." +// +kubebuilder:printcolumn:name="UPDATED_READY",type="integer",JSONPath=".status.updatedReadyReplicas",description="The number of pods ready." +// +kubebuilder:printcolumn:name="UPDATED_AVAILABLE",type="integer",JSONPath=".status.updatedAvailableReplicas",description="The number of pods updated available." +// +kubebuilder:printcolumn:name="CURRENT_REVISION",type="string",JSONPath=".status.currentRevision",description="The current revision." +// +kubebuilder:printcolumn:name="UPDATED_REVISION",type="string",JSONPath=".status.updatedRevision",description="The updated revision." +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" +// +resource:path=collasets +type CollaSet struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CollaSetSpec `json:"spec,omitempty"` + Status CollaSetStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// CollaSetList contains a list of CollaSet +type CollaSetList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CollaSet `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CollaSet{}, &CollaSetList{}) +} diff --git a/apis/apps/v1alpha1/resourcecontext_types.go b/apis/apps/v1alpha1/resourcecontext_types.go new file mode 100644 index 00000000..cd944dcc --- /dev/null +++ b/apis/apps/v1alpha1/resourcecontext_types.go @@ -0,0 +1,84 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ResourceContextSpec defines the desired state of ResourceContext +type ResourceContextSpec struct { + Contexts []ContextDetail `json:"contexts,omitempty"` +} + +type ContextDetail struct { + ID int `json:"id"` + Data map[string]string `json:"data,omitempty"` +} + +//+kubebuilder:object:root=true + +// ResourceContext is the Schema for the resourcecontext API +// +k8s:openapi-gen=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:resource:shortName=rc +type ResourceContext struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ResourceContextSpec `json:"spec,omitempty"` +} + +//+kubebuilder:object:root=true + +// ResourceContextList contains a list of ResourceContext +type ResourceContextList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ResourceContext `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ResourceContext{}, &ResourceContextList{}) +} + +// Contains is used to check whether the key-value pair in contained in Data. +func (cd *ContextDetail) Contains(key, value string) bool { + if cd.Data == nil { + return false + } + + return cd.Data[key] == value +} + +// Put is used to specify the value with the specified key in Data . +func (cd *ContextDetail) Put(key, value string) { + if cd.Data == nil { + cd.Data = map[string]string{} + } + + cd.Data[key] = value +} + +// Remove is used to remove the specified key from Data . +func (cd *ContextDetail) Remove(key string) { + if cd.Data == nil { + return + } + + delete(cd.Data, key) +} diff --git a/apis/apps/v1alpha1/well_known_annotations.go b/apis/apps/v1alpha1/well_known_annotations.go index d42545ea..7d6e86b5 100644 --- a/apis/apps/v1alpha1/well_known_annotations.go +++ b/apis/apps/v1alpha1/well_known_annotations.go @@ -17,5 +17,7 @@ limitations under the License. package v1alpha1 const ( - PodAvailableConditionsAnnotation = "cafed.kusionstack.io/available-conditions" // indicate the available conditions of a pod + PodAvailableConditionsAnnotation = "kafed.kusionstack.io/available-conditions" // indicate the available conditions of a pod + + LastPodStatusAnnotationKey = "collaset.kafed.kusionstack.io/last-pod-status" ) diff --git a/apis/apps/v1alpha1/well_known_labels.go b/apis/apps/v1alpha1/well_known_labels.go index 07bdaea5..784c6747 100644 --- a/apis/apps/v1alpha1/well_known_labels.go +++ b/apis/apps/v1alpha1/well_known_labels.go @@ -35,6 +35,11 @@ const ( PodCompleteLabelPrefix = "complete.lifecycle.kafed.kusionstack.io" // indicate a pod has finished all phases PodServiceAvailableLabel = "kafed.kusionstack.io/service-available" // indicate a pod is available to serve + + CollaSetUpdateIndicateLabelKey = "collaset.kafed.kusionstack.io/update-included" + + PodInstanceIDLabelKey = "kafed.kusionstack.io/pod-instance-id" + PodDeletionIndicationLabelKey = "kafed.kusionstack.io/to-delete" // Users can use this label to indicate a pod to delete ) var ( diff --git a/apis/apps/v1alpha1/zz_generated.deepcopy.go b/apis/apps/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 00000000..4c7d2bcc --- /dev/null +++ b/apis/apps/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,392 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ByLabel) DeepCopyInto(out *ByLabel) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ByLabel. +func (in *ByLabel) DeepCopy() *ByLabel { + if in == nil { + return nil + } + out := new(ByLabel) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ByPartition) DeepCopyInto(out *ByPartition) { + *out = *in + if in.Partition != nil { + in, out := &in.Partition, &out.Partition + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ByPartition. +func (in *ByPartition) DeepCopy() *ByPartition { + if in == nil { + return nil + } + out := new(ByPartition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CollaSet) DeepCopyInto(out *CollaSet) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CollaSet. +func (in *CollaSet) DeepCopy() *CollaSet { + if in == nil { + return nil + } + out := new(CollaSet) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CollaSet) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CollaSetCondition) DeepCopyInto(out *CollaSetCondition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CollaSetCondition. +func (in *CollaSetCondition) DeepCopy() *CollaSetCondition { + if in == nil { + return nil + } + out := new(CollaSetCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CollaSetList) DeepCopyInto(out *CollaSetList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CollaSet, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CollaSetList. +func (in *CollaSetList) DeepCopy() *CollaSetList { + if in == nil { + return nil + } + out := new(CollaSetList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CollaSetList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CollaSetSpec) DeepCopyInto(out *CollaSetSpec) { + *out = *in + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + in.Template.DeepCopyInto(&out.Template) + if in.VolumeClaimTemplates != nil { + in, out := &in.VolumeClaimTemplates, &out.VolumeClaimTemplates + *out = make([]corev1.PersistentVolumeClaim, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.UpdateStrategy.DeepCopyInto(&out.UpdateStrategy) + in.ScaleStrategy.DeepCopyInto(&out.ScaleStrategy) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CollaSetSpec. +func (in *CollaSetSpec) DeepCopy() *CollaSetSpec { + if in == nil { + return nil + } + out := new(CollaSetSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CollaSetStatus) DeepCopyInto(out *CollaSetStatus) { + *out = *in + if in.CollisionCount != nil { + in, out := &in.CollisionCount, &out.CollisionCount + *out = new(int32) + **out = **in + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]CollaSetCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CollaSetStatus. +func (in *CollaSetStatus) DeepCopy() *CollaSetStatus { + if in == nil { + return nil + } + out := new(CollaSetStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContextDetail) DeepCopyInto(out *ContextDetail) { + *out = *in + if in.Data != nil { + in, out := &in.Data, &out.Data + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContextDetail. +func (in *ContextDetail) DeepCopy() *ContextDetail { + if in == nil { + return nil + } + out := new(ContextDetail) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PersistentVolumeClaimRetentionPolicy) DeepCopyInto(out *PersistentVolumeClaimRetentionPolicy) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PersistentVolumeClaimRetentionPolicy. +func (in *PersistentVolumeClaimRetentionPolicy) DeepCopy() *PersistentVolumeClaimRetentionPolicy { + if in == nil { + return nil + } + out := new(PersistentVolumeClaimRetentionPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceContext) DeepCopyInto(out *ResourceContext) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceContext. +func (in *ResourceContext) DeepCopy() *ResourceContext { + if in == nil { + return nil + } + out := new(ResourceContext) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ResourceContext) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceContextList) DeepCopyInto(out *ResourceContextList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ResourceContext, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceContextList. +func (in *ResourceContextList) DeepCopy() *ResourceContextList { + if in == nil { + return nil + } + out := new(ResourceContextList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ResourceContextList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceContextSpec) DeepCopyInto(out *ResourceContextSpec) { + *out = *in + if in.Contexts != nil { + in, out := &in.Contexts, &out.Contexts + *out = make([]ContextDetail, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceContextSpec. +func (in *ResourceContextSpec) DeepCopy() *ResourceContextSpec { + if in == nil { + return nil + } + out := new(ResourceContextSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RollingUpdateCollaSetStrategy) DeepCopyInto(out *RollingUpdateCollaSetStrategy) { + *out = *in + if in.ByPartition != nil { + in, out := &in.ByPartition, &out.ByPartition + *out = new(ByPartition) + (*in).DeepCopyInto(*out) + } + if in.ByLabel != nil { + in, out := &in.ByLabel, &out.ByLabel + *out = new(ByLabel) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RollingUpdateCollaSetStrategy. +func (in *RollingUpdateCollaSetStrategy) DeepCopy() *RollingUpdateCollaSetStrategy { + if in == nil { + return nil + } + out := new(RollingUpdateCollaSetStrategy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ScaleStrategy) DeepCopyInto(out *ScaleStrategy) { + *out = *in + if in.PodToExclude != nil { + in, out := &in.PodToExclude, &out.PodToExclude + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PodToInclude != nil { + in, out := &in.PodToInclude, &out.PodToInclude + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PersistentVolumeClaimRetentionPolicy != nil { + in, out := &in.PersistentVolumeClaimRetentionPolicy, &out.PersistentVolumeClaimRetentionPolicy + *out = new(PersistentVolumeClaimRetentionPolicy) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScaleStrategy. +func (in *ScaleStrategy) DeepCopy() *ScaleStrategy { + if in == nil { + return nil + } + out := new(ScaleStrategy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpdateStrategy) DeepCopyInto(out *UpdateStrategy) { + *out = *in + if in.RollingUpdate != nil { + in, out := &in.RollingUpdate, &out.RollingUpdate + *out = new(RollingUpdateCollaSetStrategy) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpdateStrategy. +func (in *UpdateStrategy) DeepCopy() *UpdateStrategy { + if in == nil { + return nil + } + out := new(UpdateStrategy) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/apps.kafed.io_collasets.yaml b/config/crd/bases/apps.kafed.io_collasets.yaml new file mode 100644 index 00000000..1288c192 --- /dev/null +++ b/config/crd/bases/apps.kafed.io_collasets.yaml @@ -0,0 +1,567 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.10.0 + creationTimestamp: null + name: collasets.apps.kafed.io +spec: + group: apps.kafed.io + names: + kind: CollaSet + listKind: CollaSetList + plural: collasets + shortNames: + - cls + singular: collaset + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The desired number of pods. + jsonPath: .spec.replicas + name: DESIRED + type: integer + - description: The number of currently all pods. + jsonPath: .status.replicas + name: CURRENT + type: integer + - description: The number of pods updated. + jsonPath: .status.updatedReplicas + name: UPDATED + type: integer + - description: The number of pods ready. + jsonPath: .status.updatedReadyReplicas + name: UPDATED_READY + type: integer + - description: The number of pods updated available. + jsonPath: .status.updatedAvailableReplicas + name: UPDATED_AVAILABLE + type: integer + - description: The current revision. + jsonPath: .status.currentRevision + name: CURRENT_REVISION + type: string + - description: The updated revision. + jsonPath: .status.updatedRevision + name: UPDATED_REVISION + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: CollaSet is the Schema for the collasets API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: CollaSetSpec defines the desired state of CollaSet + properties: + historyLimit: + description: Indicate the number of histories to be conserved If unspecified, + defaults to 20 + format: int32 + type: integer + paused: + description: Indicates that the scaling and updating is paused and + will not be processed by the CollaSet controller. + type: boolean + replicas: + description: Replicas is the desired number of replicas of the given + Template. These are replicas in the sense that they are instantiations + of the same Template, but individual replicas also have a consistent + identity. If unspecified, defaults to 0. + format: int32 + type: integer + scaleStrategy: + description: ScaleStrategy indicates the strategy detail that will + be used during pod scaling. + properties: + context: + description: Context indicates the pool from which to allocate + Pod instance ID. CollaSets are allowed to share the same Context. + It is not allowed to change. Context defaults to be CollaSet's + name. + type: string + persistentVolumeClaimRetentionPolicy: + description: PersistentVolumeClaimRetentionPolicy describes the + lifecycle of PersistentVolumeClaim created from volumeClaimTemplates. + By default, all persistent volume claims are created as needed + and deleted after no pod is using them. This policy allows the + lifecycle to be altered, for example by deleting persistent + volume claims when their CollaSet is deleted, or when their + pod is scaled down. + properties: + whenDeleted: + description: WhenDeleted specifies what happens to PVCs created + from CollaSet VolumeClaimTemplates when the CollaSet is + deleted. The default policy of `Delete` policy causes those + PVCs to be deleted. `Retain` causes PVCs to not be affected + by StatefulSet deletion. The + type: string + whenScaled: + description: WhenScaled specifies what happens to PVCs created + from StatefulSet VolumeClaimTemplates when the StatefulSet + is scaled down. The default policy of `Retain` causes PVCs + to not be affected by a scaledown. The `Delete` policy causes + the associated PVCs for any excess pods above the replica + count to be deleted. + type: string + type: object + podToExclude: + description: PodToExclude indicates the pods which will be orphaned + by CollaSet. + items: + type: string + type: array + podToInclude: + description: PodToInclude indicates the pods which will be adapted + by CollaSet. + items: + type: string + type: array + type: object + selector: + description: Selector is a label query over pods that should match + the replica count. It must match the pod template's labels. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + template: + description: Template is the object that describes the pod that will + be created if insufficient replicas are detected. Each pod stamped + out by the CollaSet will fulfill this Template, but have a unique + identity from the rest of the CollaSet. + x-kubernetes-preserve-unknown-fields: true + updateStrategy: + description: UpdateStrategy indicates the CollaSetUpdateStrategy that + will be employed to update Pods in the CollaSet when a revision + is made to Template. + properties: + podUpgradePolicy: + description: PodUpdatePolicy indicates the policy by to update + pods. + type: string + rollingUpdate: + description: RollingUpdate is used to communicate parameters when + Type is RollingUpdateStatefulSetStrategyType. + properties: + byLabel: + description: ByLabel indicates the update progress is controlled + by attaching pod label. + type: object + byPartition: + description: ByPartition indicates the update progress is + controlled by partition value. + properties: + partition: + description: Partition controls the update progress by + indicating how many pods should be updated. Defaults + to nil (all pods will be updated) + format: int32 + type: integer + type: object + type: object + type: object + volumeClaimTemplates: + description: VolumeClaimTemplates is a list of claims that pods are + allowed to reference. The StatefulSet controller is responsible + for mapping network identities to claims in a way that maintains + the identity of a pod. Every claim in this list must have at least + one matching (by name) volumeMount in one container in the template. + A claim in this list takes precedence over any volumes in the template, + with the same name. + items: + description: PersistentVolumeClaim is a user's request for and claim + to a persistent volume + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this + representation of an object. Servers should convert recognized + schemas to the latest internal value, and may reject unrecognized + values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint + the client submits requests to. Cannot be updated. In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' + type: object + spec: + description: 'Spec defines the desired characteristics of a + volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'AccessModes contains the desired access modes + the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'This field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the provisioner + or an external controller can support the specified data + source, it will create a new volume based on the contents + of the specified data source. If the AnyVolumeDataSource + feature gate is enabled, this field will always have the + same contents as the DataSourceRef field.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, the + specified Kind must be in the core API group. For + any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: 'Specifies the object from which to populate + the volume with data, if a non-empty volume is desired. + This may be any local object from a non-empty API group + (non core object) or a PersistentVolumeClaim object. When + this field is specified, volume binding will only succeed + if the type of the specified object matches some installed + volume populator or dynamic provisioner. This field will + replace the functionality of the DataSource field and + as such if both fields are non-empty, they must have the + same value. For backwards compatibility, both fields (DataSource + and DataSourceRef) will be set to the same value automatically + if one of them is empty and the other is non-empty. There + are two important differences between DataSource and DataSourceRef: + * While DataSource only allows two specific types of objects, + DataSourceRef allows any non-core object, as well as PersistentVolumeClaim + objects. * While DataSource ignores disallowed values + (dropping them), DataSourceRef preserves all values, and + generates an error if a disallowed value is specified. + (Alpha) Using this field requires the AnyVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, the + specified Kind must be in the core API group. For + any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + resources: + description: 'Resources represents the minimum resources + the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of + compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is omitted + for a container, it defaults to Limits if that is + explicitly specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: A label query over volumes to consider for + binding. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: 'Name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume is required + by the claim. Value of Filesystem is implied when not + included in claim spec. + type: string + volumeName: + description: VolumeName is the binding reference to the + PersistentVolume backing this claim. + type: string + type: object + status: + description: 'Status represents the current information/status + of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'AccessModes contains the actual access modes + the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Represents the actual resources of the underlying + volume. + type: object + conditions: + description: Current Condition of persistent volume claim. + If underlying persistent volume is being resized then + the Condition will be set to 'ResizeStarted'. + items: + description: PersistentVolumeClaimCondition contails details + about state of pvc + properties: + lastProbeTime: + description: Last time we probed the condition. + format: date-time + type: string + lastTransitionTime: + description: Last time the condition transitioned + from one status to another. + format: date-time + type: string + message: + description: Human-readable message indicating details + about last transition. + type: string + reason: + description: Unique, this should be a short, machine + understandable string that gives the reason for + condition's last transition. If it reports "ResizeStarted" + that means the underlying persistent volume is being + resized. + type: string + status: + type: string + type: + description: PersistentVolumeClaimConditionType is + a valid value of PersistentVolumeClaimCondition.Type + type: string + required: + - status + - type + type: object + type: array + phase: + description: Phase represents the current phase of PersistentVolumeClaim. + type: string + type: object + type: object + type: array + type: object + status: + description: CollaSetStatus defines the observed state of CollaSet + properties: + availableReplicas: + description: The number of available replicas (ready for at least + minReadySeconds) for this replica set. + format: int32 + type: integer + collisionCount: + description: Count of hash collisions for the DaemonSet. The DaemonSet + controller uses this field as a collision avoidance mechanism when + it needs to create the name for the newest ControllerRevision. + format: int32 + type: integer + conditions: + description: Represents the latest available observations of a CollaSet's + current state. + items: + properties: + last_transition_time: + description: Last time the condition transitioned from one status + to another. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of in place set condition. + type: string + type: object + type: array + currentRevision: + description: CurrentRevision, if not empty, indicates the version + of the CollaSet. + type: string + observedGeneration: + description: ObservedGeneration is the most recent generation observed + for this CollaSet. It corresponds to the CollaSet's generation, + which is updated on mutation by the API Server. + format: int64 + type: integer + operatingReplicas: + description: OperatingReplicas indicates the number of pods during + pod ops lifecycle and not finish update-phase. + format: int32 + type: integer + readyReplicas: + description: ReadyReplicas indicates the number of the pod with ready + condition + format: int32 + type: integer + replicas: + description: Replicas is the most recently observed number of replicas. + format: int32 + type: integer + scheduledReplicas: + description: the number of scheduled replicas for the CollaSet. + format: int32 + type: integer + updatedAvailableReplicas: + description: UpdatedAvailableReplicas indicates the number of available + updated revision replicas for this CollaSet. A pod is updated available + means the pod is ready for updated revision and accessible + format: int32 + type: integer + updatedReadyReplicas: + description: UpdatedReadyReplicas indicates the number of the pod + with updated revision and ready condition + format: int32 + type: integer + updatedReplicas: + description: The number of pods in updated version. + format: int32 + type: integer + updatedRevision: + description: UpdatedRevision, if not empty, indicates the version + of the CollaSet currently updated. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/apps.kafed.io_resourcecontexts.yaml b/config/crd/bases/apps.kafed.io_resourcecontexts.yaml new file mode 100644 index 00000000..54c8a94d --- /dev/null +++ b/config/crd/bases/apps.kafed.io_resourcecontexts.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.10.0 + creationTimestamp: null + name: resourcecontexts.apps.kafed.io +spec: + group: apps.kafed.io + names: + kind: ResourceContext + listKind: ResourceContextList + plural: resourcecontexts + shortNames: + - rc + singular: resourcecontext + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ResourceContext is the Schema for the resourcecontext API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ResourceContextSpec defines the desired state of ResourceContext + properties: + contexts: + items: + properties: + data: + additionalProperties: + type: string + type: object + id: + type: integer + required: + - id + type: object + type: array + type: object + type: object + served: true + storage: true diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 269fdbcc..e2e0417d 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -2,89 +2,12 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: + creationTimestamp: null name: manager-role rules: - apiGroups: - apps.kafed.io resources: - - operationjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - apps.kafed.io - resources: - - operationjobs/finalizers - verbs: - - update -- apiGroups: - - apps.kafed.io - resources: - - operationjobs/status - verbs: - - get - - patch - - update -- apiGroups: - - apps.kafed.io - resources: - - resourcedecorations - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - apps.kafed.io - resources: - - resourcedecorations/finalizers - verbs: - - update -- apiGroups: - - apps.kafed.io - resources: - - resourcedecorations/status - verbs: - - get - - patch - - update -- apiGroups: - - apps.kafed.io - resources: - - rulesets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - apps.kafed.io - resources: - - rulesets/finalizers - verbs: - - update -- apiGroups: - - apps.kafed.io - resources: - - rulesets/status - verbs: - - get - - patch - - update -- apiGroups: - - apps.korbito.io - resources: - collasets verbs: - create @@ -95,13 +18,13 @@ rules: - update - watch - apiGroups: - - apps.korbito.io + - apps.kafed.io resources: - collasets/finalizers verbs: - update - apiGroups: - - apps.korbito.io + - apps.kafed.io resources: - collasets/status verbs: diff --git a/go.mod b/go.mod index ccd12d77..bc72d961 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module kusionstack.io/kafed go 1.19 require ( + github.com/davecgh/go-spew v1.1.1 github.com/docker/distribution v2.8.1+incompatible github.com/go-logr/logr v1.2.3 github.com/onsi/ginkgo v1.16.5 @@ -11,6 +12,7 @@ require ( github.com/prometheus/client_golang v1.14.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.1 + golang.org/x/net v0.0.0-20220225172249-27dd8689420f k8s.io/api v0.22.6 k8s.io/apimachinery v0.22.6 k8s.io/client-go v0.22.6 @@ -31,7 +33,6 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect github.com/evanphx/json-patch v4.11.0+incompatible // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect @@ -58,7 +59,6 @@ require ( go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.0 // indirect golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 // indirect - golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect diff --git a/main.go b/main.go index 777a5f25..bbe3227b 100644 --- a/main.go +++ b/main.go @@ -30,9 +30,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "kusionstack.io/kafed/apis" appsv1alpha1 "kusionstack.io/kafed/apis/apps/v1alpha1" "kusionstack.io/kafed/pkg/controllers" "kusionstack.io/kafed/pkg/utils/feature" + "kusionstack.io/kafed/pkg/utils/inject" "kusionstack.io/kafed/pkg/webhook" _ "kusionstack.io/kafed/pkg/features" @@ -83,6 +85,7 @@ func main() { LeaderElection: enableLeaderElection, LeaderElectionID: "5d84702b.kafed.io", CertDir: certDir, + NewCache: inject.NewCacheWithFieldIndex, // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily // when the Manager ends. This requires the binary to immediately end when the @@ -101,6 +104,11 @@ func main() { os.Exit(1) } + if err = apis.AddToScheme(mgr.GetScheme()); err != nil { + setupLog.Error(err, "unable to add APIs scheme") + os.Exit(1) + } + if err = controllers.AddToManager(mgr); err != nil { setupLog.Error(err, "unable to add controller") os.Exit(1) diff --git a/pkg/controllers/add_collaset.go b/pkg/controllers/add_collaset.go new file mode 100644 index 00000000..9a1b5e3e --- /dev/null +++ b/pkg/controllers/add_collaset.go @@ -0,0 +1,23 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import "kusionstack.io/kafed/pkg/controllers/collaset" + +func init() { + AddToManagerFuncs = append(AddToManagerFuncs, collaset.Add) +} diff --git a/pkg/controllers/add_poddeletion.go b/pkg/controllers/add_poddeletion.go new file mode 100644 index 00000000..dbe8a461 --- /dev/null +++ b/pkg/controllers/add_poddeletion.go @@ -0,0 +1,25 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "kusionstack.io/kafed/pkg/controllers/poddeletion" +) + +func init() { + AddToManagerFuncs = append(AddToManagerFuncs, poddeletion.Add) +} diff --git a/pkg/controllers/add_resourcecontext.go b/pkg/controllers/add_resourcecontext.go new file mode 100644 index 00000000..b2c90553 --- /dev/null +++ b/pkg/controllers/add_resourcecontext.go @@ -0,0 +1,25 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "kusionstack.io/kafed/pkg/controllers/resourcecontext" +) + +func init() { + AddToManagerFuncs = append(AddToManagerFuncs, resourcecontext.Add) +} diff --git a/pkg/controllers/collaset/collaset_controller.go b/pkg/controllers/collaset/collaset_controller.go new file mode 100644 index 00000000..541dcba9 --- /dev/null +++ b/pkg/controllers/collaset/collaset_controller.go @@ -0,0 +1,246 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collaset + +import ( + "context" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + appsv1alpha1 "kusionstack.io/kafed/apis/apps/v1alpha1" + "kusionstack.io/kafed/pkg/controllers/collaset/podcontrol" + "kusionstack.io/kafed/pkg/controllers/collaset/synccontrol" + "kusionstack.io/kafed/pkg/controllers/collaset/utils" + collasetutils "kusionstack.io/kafed/pkg/controllers/collaset/utils" + controllerutils "kusionstack.io/kafed/pkg/controllers/utils" + "kusionstack.io/kafed/pkg/controllers/utils/expectations" + "kusionstack.io/kafed/pkg/controllers/utils/podopslifecycle" + "kusionstack.io/kafed/pkg/controllers/utils/revision" +) + +const ( + controllerName = "collaset-controller" +) + +// CollaSetReconciler reconciles a CollaSet object +type CollaSetReconciler struct { + client.Client + + recorder record.EventRecorder + revisionManager *revision.RevisionManager + syncControl synccontrol.Interface +} + +func Add(mgr ctrl.Manager) error { + return AddToMgr(mgr, NewReconciler(mgr)) +} + +// NewReconciler returns a new reconcile.Reconciler +func NewReconciler(mgr ctrl.Manager) reconcile.Reconciler { + collasetutils.InitExpectations(mgr.GetClient()) + + recorder := mgr.GetEventRecorderFor(controllerName) + return &CollaSetReconciler{ + Client: mgr.GetClient(), + recorder: recorder, + revisionManager: revision.NewRevisionManager(mgr.GetClient(), mgr.GetScheme(), &revisionOwnerAdapter{}), + syncControl: synccontrol.NewRealSyncControl(mgr.GetClient(), podcontrol.NewRealPodControl(mgr.GetClient(), mgr.GetScheme()), recorder), + } +} + +func AddToMgr(mgr ctrl.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New(controllerName, mgr, controller.Options{ + MaxConcurrentReconciles: 5, + Reconciler: r, + }) + if err != nil { + return err + } + + err = c.Watch(&source.Kind{Type: &appsv1alpha1.CollaSet{}}, &handler.EnqueueRequestForObject{}) + if err != nil { + return err + } + + err = c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForOwner{ + IsController: true, + OwnerType: &appsv1alpha1.CollaSet{}, + }) + if err != nil { + return err + } + + return nil +} + +//+kubebuilder:rbac:groups=apps.kafed.io,resources=collasets,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=apps.kafed.io,resources=collasets/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=apps.kafed.io,resources=collasets/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +func (r *CollaSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + instance := &appsv1alpha1.CollaSet{} + if err := r.Get(ctx, req.NamespacedName, instance); err != nil { + if !errors.IsNotFound(err) { + klog.Error("fail to find CollaSet %s: %s", req, err) + return reconcile.Result{}, err + } + + klog.Infof("CollaSet %s is deleted", req) + return ctrl.Result{}, collasetutils.ActiveExpectations.Delete(req.Namespace, req.Name) + } + + // if expectation not satisfied, shortcut this reconciling till informer cache is updated. + if satisfied, err := collasetutils.ActiveExpectations.IsSatisfied(instance); err != nil { + return ctrl.Result{}, err + } else if !satisfied { + klog.Warningf("CollaSet %s is not satisfied to reconcile.", req) + return ctrl.Result{}, nil + } + + currentRevision, updatedRevision, revisions, collisionCount, _, err := r.revisionManager.ConstructRevisions(instance, false) + if err != nil { + return ctrl.Result{}, fmt.Errorf("fail to construct revision for CollaSet %s/%s: %s", instance.Namespace, instance.Name, err) + } + + newStatus := &appsv1alpha1.CollaSetStatus{ + // record collisionCount + CollisionCount: collisionCount, + CurrentRevision: currentRevision.Name, + UpdatedRevision: updatedRevision.Name, + } + + newStatus, err = r.DoReconcile(instance, updatedRevision, revisions, newStatus) + // update status anyway + if err := r.updateStatus(ctx, instance, newStatus); err != nil { + return ctrl.Result{}, fmt.Errorf("fail to update status of CollaSet %s: %s", req, err) + } + + return ctrl.Result{}, err +} + +func (r *CollaSetReconciler) DoReconcile(instance *appsv1alpha1.CollaSet, updatedRevision *appsv1.ControllerRevision, revisions []*appsv1.ControllerRevision, newStatus *appsv1alpha1.CollaSetStatus) (*appsv1alpha1.CollaSetStatus, error) { + podWrappers, newStatus, syncErr := r.doSync(instance, updatedRevision, revisions, newStatus) + return calculateStatus(instance, newStatus, updatedRevision, podWrappers, syncErr), syncErr +} + +// doSync is responsible for reconcile Pods with CollaSet spec. +// 1. sync Pods to prepare information, especially IDs, for following Scale and Update +// 2. scale Pods to match the Pod number indicated in `spec.replcas`. if an error thrown out or Pods is not matched recently, update will be skipped. +// 3. update Pods, to update each Pod to the updated revision indicated by `spec.template` +func (r *CollaSetReconciler) doSync(instance *appsv1alpha1.CollaSet, updatedRevision *appsv1.ControllerRevision, revisions []*appsv1.ControllerRevision, newStatus *appsv1alpha1.CollaSetStatus) ([]*collasetutils.PodWrapper, *appsv1alpha1.CollaSetStatus, error) { + synced, podWrappers, ownedIDs, err := r.syncControl.SyncPods(instance, updatedRevision, newStatus) + if err != nil || synced { + return podWrappers, newStatus, err + } + + scaling, err := r.syncControl.Scale(instance, podWrappers, revisions, updatedRevision, ownedIDs, newStatus) + if err != nil || scaling { + return podWrappers, newStatus, err + } + + _, err = r.syncControl.Update(instance, podWrappers, revisions, updatedRevision, ownedIDs, newStatus) + + return podWrappers, newStatus, err +} + +func calculateStatus(instance *appsv1alpha1.CollaSet, newStatus *appsv1alpha1.CollaSetStatus, updatedRevision *appsv1.ControllerRevision, podWrappers []*collasetutils.PodWrapper, syncErr error) *appsv1alpha1.CollaSetStatus { + if syncErr == nil { + newStatus.ObservedGeneration = instance.Generation + } + + var scheduledReplicas, readyReplicas, availableReplicas, replicas, updatedReplicas, operatingReplicas, + updatedReadyReplicas, updatedAvailableReplicas int32 + + for _, podWrapper := range podWrappers { + replicas++ + + isUpdated := false + if isUpdated = controllerutils.IsPodUpdatedRevision(podWrapper.Pod, updatedRevision.Name); isUpdated { + updatedReplicas++ + } + + if podopslifecycle.IsDuringOps(utils.UpdateOpsLifecycleAdapter, podWrapper) || podopslifecycle.IsDuringOps(utils.ScaleInOpsLifecycleAdapter, podWrapper) { + operatingReplicas++ + } + + if controllerutils.IsPodScheduled(podWrapper.Pod) { + scheduledReplicas++ + } + + if controllerutils.IsPodReady(podWrapper.Pod) { + readyReplicas++ + if isUpdated { + updatedReadyReplicas++ + } + } + + if controllerutils.IsServiceAvailable(podWrapper.Pod) { + availableReplicas++ + if isUpdated { + updatedAvailableReplicas++ + } + } + } + + newStatus.ScheduledReplicas = scheduledReplicas + newStatus.ReadyReplicas = readyReplicas + newStatus.AvailableReplicas = availableReplicas + newStatus.Replicas = replicas + newStatus.UpdatedReplicas = updatedReplicas + newStatus.OperatingReplicas = operatingReplicas + newStatus.UpdatedReadyReplicas = updatedReadyReplicas + newStatus.UpdatedAvailableReplicas = updatedAvailableReplicas + + if (instance.Spec.Replicas == nil && newStatus.UpdatedAvailableReplicas == 0) || + *instance.Spec.Replicas == newStatus.UpdatedAvailableReplicas { + newStatus.CurrentRevision = updatedRevision.Name + } + + return newStatus +} + +func (r *CollaSetReconciler) updateStatus(ctx context.Context, instance *appsv1alpha1.CollaSet, newStatus *appsv1alpha1.CollaSetStatus) error { + if equality.Semantic.DeepEqual(instance.Status, newStatus) { + return nil + } + + instance.Status = *newStatus + + err := r.Status().Update(ctx, instance) + if err == nil { + if err := collasetutils.ActiveExpectations.ExpectUpdate(instance, expectations.CollaSet, instance.Name, instance.ResourceVersion); err != nil { + return err + } + } + + return err +} diff --git a/pkg/controllers/collaset/collaset_controller_test.go b/pkg/controllers/collaset/collaset_controller_test.go new file mode 100644 index 00000000..b40ab2f9 --- /dev/null +++ b/pkg/controllers/collaset/collaset_controller_test.go @@ -0,0 +1,779 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collaset + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "kusionstack.io/kafed/apis" + appsv1alpha1 "kusionstack.io/kafed/apis/apps/v1alpha1" + "kusionstack.io/kafed/pkg/controllers/collaset/synccontrol" + collasetutils "kusionstack.io/kafed/pkg/controllers/collaset/utils" + "kusionstack.io/kafed/pkg/controllers/utils/podopslifecycle" + "kusionstack.io/kafed/pkg/utils/inject" +) + +var ( + env *envtest.Environment + mgr manager.Manager + request chan reconcile.Request + + ctx context.Context + cancel context.CancelFunc + c client.Client +) + +var _ = Describe("collaset controller", func() { + + It("scale reconcile", func() { + testcase := "test-scale" + Expect(createNamespace(c, testcase)).Should(BeNil()) + + cs := &appsv1alpha1.CollaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: "foo", + }, + Spec: appsv1alpha1.CollaSetSpec{ + Replicas: int32Pointer(2), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "foo", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "foo", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + + Expect(c.Create(context.TODO(), cs)).Should(BeNil()) + + podList := &corev1.PodList{} + Eventually(func() bool { + Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + return len(podList.Items) == 2 + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}, cs)).Should(BeNil()) + Expect(expectedStatusReplicas(c, cs, 0, 0, 0, 2, 2, 0, 0, 0)).Should(BeNil()) + + // pod replica will be kept, after deleted + podToDelete := &podList.Items[0] + // add finalizer to block its deletion + Expect(updatePodWithRetry(c, podToDelete.Namespace, podToDelete.Name, func(pod *corev1.Pod) bool { + pod.Finalizers = append(pod.Finalizers, "block/deletion") + return true + })).Should(BeNil()) + Expect(c.Delete(context.TODO(), podToDelete)).Should(BeNil()) + Eventually(func() bool { + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: podToDelete.Namespace, Name: podToDelete.Name}, podToDelete)).Should(BeNil()) + return podToDelete.DeletionTimestamp != nil + }, 500*time.Second, 1*time.Second).Should(BeTrue()) + + // there should be 3 pods and one of them is terminating + Eventually(func() bool { + Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + return len(podList.Items) == 3 + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + Expect(len(podList.Items)).Should(BeEquivalentTo(3)) + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}, cs)).Should(BeNil()) + Expect(expectedStatusReplicas(c, cs, 0, 0, 0, 2, 2, 0, 0, 0)).Should(BeNil()) + // should use Pod instance ID: 0, 1 + podInstanceID := sets.Int{} + podNames := sets.String{} + for _, pod := range podList.Items { + id, err := collasetutils.GetPodInstanceID(&pod) + Expect(err).Should(BeNil()) + podInstanceID.Insert(id) + podNames.Insert(pod.Name) + } + Expect(podInstanceID.Len()).Should(BeEquivalentTo(2)) + Expect(podInstanceID.Has(0)).Should(BeTrue()) + Expect(podInstanceID.Has(1)).Should(BeTrue()) + Expect(podNames.Has(podToDelete.Name)).Should(BeTrue()) + // let terminating Pod disappeared + Expect(updatePodWithRetry(c, podToDelete.Namespace, podToDelete.Name, func(pod *corev1.Pod) bool { + pod.Finalizers = []string{} + return true + })).Should(BeNil()) + Eventually(func() bool { + return c.Get(context.TODO(), types.NamespacedName{Namespace: podToDelete.Namespace, Name: podToDelete.Name}, podToDelete) != nil + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + + // scale in pods + Expect(updateCollaSetWithRetry(c, cs.Namespace, cs.Name, func(cls *appsv1alpha1.CollaSet) bool { + cls.Spec.Replicas = int32Pointer(0) + return true + })).Should(BeNil()) + + // mark all pods allowed to operate in PodOpsLifecycle + Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + Expect(len(podList.Items)).Should(BeEquivalentTo(2)) + for i := range podList.Items { + pod := &podList.Items[i] + Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.ScaleInOpsLifecycleAdapter.GetID()) + pod.Labels[labelOperate] = "true" + return true + })).Should(BeNil()) + } + + Eventually(func() bool { + Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + return len(podList.Items) == 0 + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}, cs)).Should(BeNil()) + Expect(expectedStatusReplicas(c, cs, 0, 0, 0, 0, 0, 0, 0, 0)).Should(BeNil()) + }) + + It("update reconcile", func() { + testcase := "test-update" + Expect(createNamespace(c, testcase)).Should(BeNil()) + + cs := &appsv1alpha1.CollaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: "foo", + }, + Spec: appsv1alpha1.CollaSetSpec{ + Replicas: int32Pointer(4), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "foo", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "foo", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + + Expect(c.Create(context.TODO(), cs)).Should(BeNil()) + + podList := &corev1.PodList{} + Eventually(func() bool { + Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + return len(podList.Items) == 4 + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}, cs)).Should(BeNil()) + Expect(expectedStatusReplicas(c, cs, 0, 0, 0, 4, 4, 0, 0, 0)).Should(BeNil()) + + // test update ByPartition + for _, partition := range []int32{0, 1, 2, 3, 4} { + observedGeneration := cs.Status.ObservedGeneration + + // update CollaSet image and partition + Expect(updateCollaSetWithRetry(c, cs.Namespace, cs.Name, func(cls *appsv1alpha1.CollaSet) bool { + cls.Spec.UpdateStrategy.RollingUpdate = &appsv1alpha1.RollingUpdateCollaSetStrategy{ + ByPartition: &appsv1alpha1.ByPartition{ + Partition: &partition, + }, + } + cls.Spec.Template.Spec.Containers[0].Image = "nginx:v2" + + return true + })).Should(BeNil()) + Eventually(func() bool { + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}, cs)).Should(BeNil()) + return cs.Status.ObservedGeneration != observedGeneration + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + + podList := &corev1.PodList{} + Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + for i := range podList.Items { + pod := &podList.Items[i] + if !podopslifecycle.IsDuringOps(collasetutils.UpdateOpsLifecycleAdapter, pod) { + continue + } + + if podopslifecycle.AllowOps(collasetutils.UpdateOpsLifecycleAdapter, pod) { + continue + } + // allow Pod to do update + Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + pod.Labels[labelOperate] = "true" + return true + })).Should(BeNil()) + } + // check updated pod replicas by CollaSet status + Eventually(func() error { + return expectedStatusReplicas(c, cs, 0, 0, 0, 4, partition, partition, 0, 0) + }, 5*time.Second, 1*time.Second).Should(BeNil()) + + // double check updated pod replicas + Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + updatedReplicas := 0 + for _, pod := range podList.Items { + if pod.Spec.Containers[0].Image == cs.Spec.Template.Spec.Containers[0].Image { + updatedReplicas++ + Expect(pod.Annotations).ShouldNot(BeNil()) + Expect(pod.Annotations[appsv1alpha1.LastPodStatusAnnotationKey]).ShouldNot(BeEquivalentTo("")) + + podStatus := &synccontrol.PodStatus{} + Expect(json.Unmarshal([]byte(pod.Annotations[appsv1alpha1.LastPodStatusAnnotationKey]), podStatus)).Should(BeNil()) + Expect(len(podStatus.ContainerStates)).Should(BeEquivalentTo(1)) + Expect(podStatus.ContainerStates["foo"].LatestImage).Should(BeEquivalentTo("nginx:v2")) + } + } + Expect(updatedReplicas).Should(BeEquivalentTo(partition)) + } + + // mock Pods updated by kubelet + Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + for i := range podList.Items { + Expect(updatePodStatusWithRetry(c, podList.Items[i].Namespace, podList.Items[i].Name, func(pod *corev1.Pod) bool { + pod.Status.ContainerStatuses = []corev1.ContainerStatus{ + { + Name: "foo", + ImageID: "id:v2", + }, + } + return true + })).Should(BeNil()) + } + + Eventually(func() error { + return expectedStatusReplicas(c, cs, 0, 0, 0, 4, 4, 0, 0, 0) + }, 5*time.Second, 1*time.Second).Should(BeNil()) + + // test update ByLabel + observedGeneration := cs.Status.ObservedGeneration + // update CollaSet image + Expect(updateCollaSetWithRetry(c, cs.Namespace, cs.Name, func(cls *appsv1alpha1.CollaSet) bool { + cls.Spec.UpdateStrategy.RollingUpdate = &appsv1alpha1.RollingUpdateCollaSetStrategy{ + ByLabel: &appsv1alpha1.ByLabel{}, + } + cls.Spec.Template.Spec.Containers[0].Image = "nginx:v3" + + return true + })).Should(BeNil()) + Eventually(func() bool { + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}, cs)).Should(BeNil()) + return cs.Status.ObservedGeneration != observedGeneration + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + + Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + for _, number := range []int32{1, 2, 3, 4} { + pod := podList.Items[number-1] + // label pod to trigger update + Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { + if pod.Labels == nil { + pod.Labels = map[string]string{} + } + pod.Labels[appsv1alpha1.CollaSetUpdateIndicateLabelKey] = "true" + return true + })).Should(BeNil()) + + Eventually(func() error { + // check updated pod replicas by CollaSet status + return expectedStatusReplicas(c, cs, 0, 0, 0, 4, number, 1, 0, 0) + }, 5*time.Second, 1*time.Second).Should(BeNil()) + + Expect(updatePodStatusWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { + pod.Status.ContainerStatuses = []corev1.ContainerStatus{ + { + Name: "foo", + ImageID: "id:v3", + }, + } + return true + })).Should(BeNil()) + + // double check updated pod replicas + podList := &corev1.PodList{} + Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + updatedReplicas := 0 + for _, pod := range podList.Items { + Eventually(func() bool { + tmpPod := &corev1.Pod{} + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, tmpPod)).Should(BeNil()) + return tmpPod.Labels[appsv1alpha1.CollaSetUpdateIndicateLabelKey] == "" + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + if pod.Spec.Containers[0].Image == cs.Spec.Template.Spec.Containers[0].Image { + updatedReplicas++ + } + } + Expect(updatedReplicas).Should(BeEquivalentTo(number)) + + Eventually(func() error { + // check updated pod replicas by CollaSet status + return expectedStatusReplicas(c, cs, 0, 0, 0, 4, number, 0, 0, 0) + }, 5*time.Second, 1*time.Second).Should(BeNil()) + } + }) + + It("update pod policy", func() { + testcase := "test-update-pod-policy" + Expect(createNamespace(c, testcase)).Should(BeNil()) + + cs := &appsv1alpha1.CollaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: "foo", + }, + Spec: appsv1alpha1.CollaSetSpec{ + Replicas: int32Pointer(1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "foo", + "test1": "v1", + "test2": "v1", + "test3": "v1", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "foo", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + + Expect(c.Create(context.TODO(), cs)).Should(BeNil()) + + podList := &corev1.PodList{} + Eventually(func() bool { + Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + return len(podList.Items) == 1 + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}, cs)).Should(BeNil()) + Expect(expectedStatusReplicas(c, cs, 0, 0, 0, 1, 1, 0, 0, 0)).Should(BeNil()) + + pod := podList.Items[0] + // test attribute updated manually + Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { + if pod.Labels == nil { + pod.Labels = map[string]string{} + } + pod.Labels["test1"] = "v2" + pod.Labels["manual-added"] = "v1" + delete(pod.Labels, "test2") + return true + })).Should(BeNil()) + + observedGeneration := cs.Status.ObservedGeneration + // update CollaSet image + Expect(updateCollaSetWithRetry(c, cs.Namespace, cs.Name, func(cls *appsv1alpha1.CollaSet) bool { + cls.Spec.Template.Labels["test1"] = "v3" + delete(cls.Spec.Template.Labels, "test3") + return true + })).Should(BeNil()) + + // allow Pod to update + Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + pod.Labels[labelOperate] = "true" + return true + })).Should(BeNil()) + + Eventually(func() bool { + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}, cs)).Should(BeNil()) + return cs.Status.ObservedGeneration != observedGeneration && cs.Status.UpdatedReplicas == 1 + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + + // check pod label + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, &pod)).Should(BeNil()) + Expect(pod.Labels).ShouldNot(BeNil()) + // manual changes should have lower priority than PodTemplate changes + Expect(pod.Labels["test1"]).Should(BeEquivalentTo("v3")) + Expect(pod.Labels["manual-added"]).Should(BeEquivalentTo("v1")) + + _, exist := pod.Labels["test2"] + Expect(exist).Should(BeFalse()) + _, exist = pod.Labels["test3"] + Expect(exist).Should(BeFalse()) + + // test Recreate policy + observedGeneration = cs.Status.ObservedGeneration + // update CollaSet image + Expect(updateCollaSetWithRetry(c, cs.Namespace, cs.Name, func(cls *appsv1alpha1.CollaSet) bool { + cls.Spec.Template.Labels["test1"] = "v4" + cls.Spec.UpdateStrategy.PodUpdatePolicy = appsv1alpha1.CollaSetRecreatePodUpdateStrategyType + return true + })).Should(BeNil()) + Eventually(func() bool { + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}, cs)).Should(BeNil()) + return cs.Status.ObservedGeneration != observedGeneration && cs.Status.UpdatedReplicas == 1 + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + + // pod should be recreated + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, &pod)).ShouldNot(BeNil()) + + Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + Expect(len(podList.Items)).Should(BeEquivalentTo(1)) + pod = podList.Items[0] + // check pod label + Expect(pod.Labels).ShouldNot(BeNil()) + // manual changes should have lower priority than PodTemplate changes + Expect(pod.Labels["test1"]).Should(BeEquivalentTo("v4")) + Expect(pod.Labels["test2"]).Should(BeEquivalentTo("v1")) + + _, exist = pod.Labels["manual-added"] + Expect(exist).Should(BeFalse()) + _, exist = pod.Labels["test3"] + Expect(exist).Should(BeFalse()) + }) + + It("pod recreate with its current revision", func() { + testcase := "test-pod-recreate-with-its-revision" + Expect(createNamespace(c, testcase)).Should(BeNil()) + + cs := &appsv1alpha1.CollaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: "foo", + }, + Spec: appsv1alpha1.CollaSetSpec{ + Replicas: int32Pointer(2), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "foo", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "foo", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + + Expect(c.Create(context.TODO(), cs)).Should(BeNil()) + + podList := &corev1.PodList{} + Eventually(func() bool { + Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + return len(podList.Items) == 2 + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}, cs)).Should(BeNil()) + Expect(expectedStatusReplicas(c, cs, 0, 0, 0, 2, 2, 0, 0, 0)).Should(BeNil()) + + var partition int32 = 1 + Expect(updateCollaSetWithRetry(c, cs.Namespace, cs.Name, func(cls *appsv1alpha1.CollaSet) bool { + cls.Spec.Template.Spec.Containers[0].Image = "nginx:v2" + cls.Spec.UpdateStrategy.RollingUpdate = &appsv1alpha1.RollingUpdateCollaSetStrategy{ + ByPartition: &appsv1alpha1.ByPartition{ + Partition: &partition, + }, + } + return true + })) + + for i := range podList.Items { + pod := &podList.Items[i] + // allow Pod to update + Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + pod.Labels[labelOperate] = "true" + return true + })).Should(BeNil()) + } + Eventually(func() error { + // check updated pod replicas by CollaSet status + return expectedStatusReplicas(c, cs, 0, 0, 0, 2, 1, 1, 0, 0) + }, 5*time.Second, 1*time.Second).Should(BeNil()) + + idToRevision := map[int]string{} + Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + Expect(len(podList.Items)).Should(BeEquivalentTo(2)) + for i := range podList.Items { + pod := &podList.Items[i] + id, _ := collasetutils.GetPodInstanceID(pod) + Expect(id >= 0).Should(BeTrue()) + revision := pod.Labels[appsv1.ControllerRevisionHashLabelKey] + Expect(revision).ShouldNot(BeEquivalentTo("")) + idToRevision[id] = revision + } + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}, cs)).Should(BeNil()) + + // delete pod + for i := range podList.Items { + Expect(c.Delete(context.TODO(), &podList.Items[i])).Should(BeNil()) + } + + Eventually(func() int { + Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + return len(podList.Items) + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) + Eventually(func() error { + // check updated pod replicas by CollaSet status + return expectedStatusReplicas(c, cs, 0, 0, 0, 2, 1, 0, 0, 0) + }, 5*time.Second, 1*time.Second).Should(BeNil()) + for i := range podList.Items { + pod := &podList.Items[i] + id, _ := collasetutils.GetPodInstanceID(pod) + Expect(id >= 0).Should(BeTrue()) + revision := pod.Labels[appsv1.ControllerRevisionHashLabelKey] + Expect(revision).ShouldNot(BeEquivalentTo("")) + Expect(idToRevision[id]).Should(BeEquivalentTo(revision)) + } + }) +}) + +func expectedStatusReplicas(c client.Client, cls *appsv1alpha1.CollaSet, scheduledReplicas, readyReplicas, availableReplicas, replicas, updatedReplicas, operatingReplicas, + updatedReadyReplicas, updatedAvailableReplicas int32) error { + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: cls.Namespace, Name: cls.Name}, cls); err != nil { + return err + } + + if cls.Status.ScheduledReplicas != scheduledReplicas { + return fmt.Errorf("scheduledReplicas got %d, expected %d", cls.Status.ScheduledReplicas, scheduledReplicas) + } + + if cls.Status.ReadyReplicas != readyReplicas { + return fmt.Errorf("readyReplicas got %d, expected %d", cls.Status.ReadyReplicas, readyReplicas) + } + + if cls.Status.AvailableReplicas != availableReplicas { + return fmt.Errorf("availableReplicas got %d, expected %d", cls.Status.AvailableReplicas, availableReplicas) + } + + if cls.Status.Replicas != replicas { + return fmt.Errorf("replicas got %d, expected %d", cls.Status.Replicas, replicas) + } + + if cls.Status.UpdatedReplicas != updatedReplicas { + return fmt.Errorf("updatedReplicas got %d, expected %d", cls.Status.UpdatedReplicas, updatedReplicas) + } + + if cls.Status.OperatingReplicas != operatingReplicas { + return fmt.Errorf("operatingReplicas got %d, expected %d", cls.Status.OperatingReplicas, operatingReplicas) + } + + if cls.Status.UpdatedReadyReplicas != updatedReadyReplicas { + return fmt.Errorf("updatedReadyReplicas got %d, expected %d", cls.Status.UpdatedReadyReplicas, updatedReadyReplicas) + } + + if cls.Status.UpdatedAvailableReplicas != updatedAvailableReplicas { + return fmt.Errorf("updatedAvailableReplicas got %d, expected %d", cls.Status.UpdatedAvailableReplicas, updatedAvailableReplicas) + } + + return nil +} + +func updateCollaSetWithRetry(c client.Client, namespace, name string, updateFn func(cls *appsv1alpha1.CollaSet) bool) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + cls := &appsv1alpha1.CollaSet{} + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: name}, cls); err != nil { + return err + } + + if !updateFn(cls) { + return nil + } + + return c.Update(context.TODO(), cls) + }) +} + +func updatePodWithRetry(c client.Client, namespace, name string, updateFn func(pod *corev1.Pod) bool) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + pod := &corev1.Pod{} + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: name}, pod); err != nil { + return err + } + + if !updateFn(pod) { + return nil + } + + return c.Update(context.TODO(), pod) + }) +} + +func updatePodStatusWithRetry(c client.Client, namespace, name string, updateFn func(pod *corev1.Pod) bool) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + pod := &corev1.Pod{} + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: name}, pod); err != nil { + return err + } + + if !updateFn(pod) { + return nil + } + + return c.Status().Update(context.TODO(), pod) + }) +} + +func testReconcile(inner reconcile.Reconciler) (reconcile.Reconciler, chan reconcile.Request) { + requests := make(chan reconcile.Request, 5) + fn := reconcile.Func(func(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + result, err := inner.Reconcile(ctx, req) + if _, done := ctx.Deadline(); !done && len(requests) == 0 { + requests <- req + } + return result, err + }) + return fn, requests +} + +func TestCollaSetController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "CollaSetController Test Suite") +} + +var _ = BeforeSuite(func() { + By("bootstrapping test environment") + + ctx, cancel = context.WithCancel(context.TODO()) + logf.SetLogger(zap.New(zap.WriteTo(os.Stdout), zap.UseDevMode(true))) + + env = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, + } + env.ControlPlane.GetAPIServer().URL = &url.URL{ + Host: "127.0.0.1:10001", + } + config, err := env.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(config).NotTo(BeNil()) + + mgr, err = manager.New(config, manager.Options{ + MetricsBindAddress: "0", + NewCache: inject.NewCacheWithFieldIndex, + }) + Expect(err).NotTo(HaveOccurred()) + + scheme := mgr.GetScheme() + err = appsv1.SchemeBuilder.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + err = apis.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + c = mgr.GetClient() + + var r reconcile.Reconciler + r, request = testReconcile(NewReconciler(mgr)) + err = AddToMgr(mgr, r) + Expect(err).NotTo(HaveOccurred()) + + go func() { + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }() +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + + cancel() + + err := env.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +var _ = AfterEach(func() { + csList := &appsv1alpha1.CollaSetList{} + Expect(mgr.GetClient().List(context.Background(), csList)).Should(BeNil()) + + for i := range csList.Items { + Expect(mgr.GetClient().Delete(context.TODO(), &csList.Items[i])).Should(BeNil()) + } + + nsList := &corev1.NamespaceList{} + Expect(mgr.GetClient().List(context.Background(), nsList)).Should(BeNil()) + + for i := range nsList.Items { + if strings.HasPrefix(nsList.Items[i].Name, "test-") { + mgr.GetClient().Delete(context.TODO(), &nsList.Items[i]) + } + } +}) + +func createNamespace(c client.Client, namespaceName string) error { + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespaceName, + }, + } + + return c.Create(context.TODO(), ns) +} + +func int32Pointer(val int32) *int32 { + return &val +} diff --git a/pkg/controllers/collaset/podcontext/podcontext.go b/pkg/controllers/collaset/podcontext/podcontext.go new file mode 100644 index 00000000..76f1b519 --- /dev/null +++ b/pkg/controllers/collaset/podcontext/podcontext.go @@ -0,0 +1,191 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podcontext + +import ( + "context" + "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "kusionstack.io/kafed/pkg/controllers/collaset/utils" + "sort" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "kusionstack.io/kafed/apis/apps/v1alpha1" + "kusionstack.io/kafed/pkg/controllers/utils/expectations" +) + +const ( + OwnerContextKey = "Owner" + RevisionContextDataKey = "Revision" +) + +func AllocateID(c client.Client, instance *appsv1alpha1.CollaSet, defaultRevision string, replicas int) (map[int]*appsv1alpha1.ContextDetail, error) { + contextName := getContextName(instance) + podContext := &appsv1alpha1.ResourceContext{} + notFound := false + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: instance.Namespace, Name: contextName}, podContext); err != nil { + if !errors.IsNotFound(err) { + return nil, fmt.Errorf("fail to find ResourceContext %s/%s for owner %s: %s", instance.Namespace, contextName, instance.Name, err) + } + + notFound = true + podContext.Namespace = instance.Namespace + podContext.Name = contextName + } + + // store all the IDs crossing Multiple workload + existingIDs := map[int]*appsv1alpha1.ContextDetail{} + // only store the IDs belonging to this owner + ownedIDs := map[int]*appsv1alpha1.ContextDetail{} + for i := range podContext.Spec.Contexts { + detail := &podContext.Spec.Contexts[i] + if detail.Contains(OwnerContextKey, instance.Name) { + ownedIDs[detail.ID] = detail + } + + existingIDs[detail.ID] = detail + } + + // if owner has enough ID, return + if len(ownedIDs) >= replicas { + return ownedIDs, nil + } + + // find new IDs for owner + candidateID := 0 + for len(ownedIDs) < replicas { + // find one new ID + for { + if _, exist := existingIDs[candidateID]; exist { + candidateID++ + continue + } + + break + } + + detail := &appsv1alpha1.ContextDetail{ + ID: candidateID, + Data: map[string]string{ + OwnerContextKey: instance.Name, + RevisionContextDataKey: defaultRevision, + }, + } + existingIDs[candidateID] = detail + ownedIDs[candidateID] = detail + } + + if notFound { + return ownedIDs, doCreatePodContext(c, instance, ownedIDs) + } + + return ownedIDs, doUpdatePodContext(c, instance, ownedIDs, podContext) +} + +func UpdateToPodContext(c client.Client, instance *appsv1alpha1.CollaSet, ownedIDs map[int]*appsv1alpha1.ContextDetail) error { + contextName := getContextName(instance) + podContext := &appsv1alpha1.ResourceContext{} + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: instance.Namespace, Name: contextName}, podContext); err != nil { + if !errors.IsNotFound(err) { + return fmt.Errorf("fail to find ResourceContext %s/%s: %s", instance.Namespace, contextName, err) + } + + if err := doCreatePodContext(c, instance, ownedIDs); err != nil { + return fmt.Errorf("fail to create ResourceContext %s/%s after not found: %s", instance.Namespace, contextName, err) + } + } + + return doUpdatePodContext(c, instance, ownedIDs, podContext) +} + +func doCreatePodContext(c client.Client, instance *appsv1alpha1.CollaSet, ownerIDs map[int]*appsv1alpha1.ContextDetail) error { + contextName := getContextName(instance) + podContext := &appsv1alpha1.ResourceContext{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: instance.Namespace, + Name: contextName, + }, + Spec: appsv1alpha1.ResourceContextSpec{ + Contexts: make([]appsv1alpha1.ContextDetail, len(ownerIDs)), + }, + } + + i := 0 + for _, detail := range ownerIDs { + podContext.Spec.Contexts[i] = *detail + i++ + } + + return c.Create(context.TODO(), podContext) +} + +func doUpdatePodContext(c client.Client, instance client.Object, ownedIDs map[int]*appsv1alpha1.ContextDetail, podContext *appsv1alpha1.ResourceContext) error { + // store all IDs crossing all workload + existingIDs := map[int]*appsv1alpha1.ContextDetail{} + for k, detail := range ownedIDs { + existingIDs[k] = detail + } + + for i := range podContext.Spec.Contexts { + detail := podContext.Spec.Contexts[i] + if detail.Contains(OwnerContextKey, instance.GetName()) { + continue + } + + existingIDs[detail.ID] = &detail + } + + podContext.Spec.Contexts = make([]appsv1alpha1.ContextDetail, len(existingIDs)) + + idx := 0 + for _, contextDetail := range existingIDs { + podContext.Spec.Contexts[idx] = *contextDetail + idx++ + } + + // keep context detail in order by ID + sort.Sort(ContextDetailsByOrder(podContext.Spec.Contexts)) + err := c.Update(context.TODO(), podContext) + if err != nil { + if err := utils.ActiveExpectations.ExpectUpdate(instance, expectations.ResourceContext, podContext.Name, podContext.ResourceVersion); err != nil { + return err + } + } + + return err +} + +func getContextName(instance *appsv1alpha1.CollaSet) string { + if instance.Spec.ScaleStrategy.Context != "" { + return instance.Spec.ScaleStrategy.Context + } + + return instance.Name +} + +type ContextDetailsByOrder []appsv1alpha1.ContextDetail + +func (s ContextDetailsByOrder) Len() int { return len(s) } +func (s ContextDetailsByOrder) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func (s ContextDetailsByOrder) Less(i, j int) bool { + l, r := s[i], s[j] + return l.ID < r.ID +} diff --git a/pkg/controllers/collaset/podcontext/podcontext_test.go b/pkg/controllers/collaset/podcontext/podcontext_test.go new file mode 100644 index 00000000..677bb93e --- /dev/null +++ b/pkg/controllers/collaset/podcontext/podcontext_test.go @@ -0,0 +1,115 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podcontext + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + appsv1alpha1 "kusionstack.io/kafed/apis/apps/v1alpha1" +) + +func init() { + corev1.AddToScheme(scheme) + appsv1alpha1.AddToScheme(scheme) +} + +var ( + scheme = runtime.NewScheme() +) + +var _ = Describe("ResourceContext allocation", func() { + + It("allocate ID", func() { + c := fake.NewClientBuilder().WithScheme(scheme).Build() + namespace := "test" + + instance1 := &appsv1alpha1.CollaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "foo1", + }, + Spec: appsv1alpha1.CollaSetSpec{ + ScaleStrategy: appsv1alpha1.ScaleStrategy{ + Context: "foo", // use the same Context + }, + }, + } + + instance2 := &appsv1alpha1.CollaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "foo2", + }, + Spec: appsv1alpha1.CollaSetSpec{ + ScaleStrategy: appsv1alpha1.ScaleStrategy{ + Context: "foo", // use the same Context + }, + }, + } + + ownedIDs, err := AllocateID(c, instance1, "", 10) + Expect(err).Should(BeNil()) + Expect(len(ownedIDs)).Should(BeEquivalentTo(10)) + for i := 0; i < 10; i++ { + _, exist := ownedIDs[i] + Expect(exist).Should(BeTrue()) + } + + ownedIDs, err = AllocateID(c, instance2, "", 4) + Expect(err).Should(BeNil()) + Expect(len(ownedIDs)).Should(BeEquivalentTo(4)) + for i := 10; i < 14; i++ { + _, exist := ownedIDs[i] + Expect(exist).Should(BeTrue()) + } + + ownedIDs, err = AllocateID(c, instance1, "", 10) + delete(ownedIDs, 4) + delete(ownedIDs, 6) + Expect(UpdateToPodContext(c, instance1, ownedIDs)).Should(BeNil()) + + ownedIDs, err = AllocateID(c, instance2, "", 7) + Expect(err).Should(BeNil()) + Expect(len(ownedIDs)).Should(BeEquivalentTo(7)) + for _, i := range []int{4, 6, 10, 11, 12, 13, 14} { + _, exist := ownedIDs[i] + Expect(exist).Should(BeTrue()) + } + + ownedIDs, err = AllocateID(c, instance1, "", 12) + Expect(err).Should(BeNil()) + Expect(len(ownedIDs)).Should(BeEquivalentTo(12)) + for _, i := range []int{0, 1, 2, 3, 5, 7, 8, 9, 15, 16, 17, 18} { + _, exist := ownedIDs[i] + Expect(exist).Should(BeTrue()) + } + }) + +}) + +func TestPodContext(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "ResourceContext Test Suite") +} diff --git a/pkg/controllers/collaset/podcontrol/pod_control.go b/pkg/controllers/collaset/podcontrol/pod_control.go new file mode 100644 index 00000000..bbf9b6f9 --- /dev/null +++ b/pkg/controllers/collaset/podcontrol/pod_control.go @@ -0,0 +1,126 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podcontrol + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + refmanagerutil "kusionstack.io/kafed/pkg/controllers/utils/refmanager" + "kusionstack.io/kafed/pkg/utils/inject" +) + +type Interface interface { + GetFilteredPods(selector *metav1.LabelSelector, owner client.Object) ([]*corev1.Pod, error) + CreatePod(pod *corev1.Pod) (*corev1.Pod, error) + DeletePod(pod *corev1.Pod) error + UpdatePod(pod *corev1.Pod) error +} + +func NewRealPodControl(client client.Client, scheme *runtime.Scheme) Interface { + return &RealPodControl{ + client: client, + scheme: scheme, + } +} + +type RealPodControl struct { + client client.Client + scheme *runtime.Scheme +} + +func (pc *RealPodControl) GetFilteredPods(selector *metav1.LabelSelector, owner client.Object) ([]*corev1.Pod, error) { + // get the pods with ownerReference points to this CollaSet + podList := &corev1.PodList{} + err := pc.client.List(context.TODO(), podList, &client.ListOptions{Namespace: owner.GetNamespace(), + FieldSelector: fields.OneTermEqualSelector(inject.FieldIndexOwnerRefUID, string(owner.GetUID()))}) + if err != nil { + return nil, err + } + + filteredPods := FilterOutInactivePod(podList.Items) + filteredPods, err = pc.getPodSetPods(filteredPods, selector, owner) + if err != nil { + return nil, err + } + + return filteredPods, nil +} + +func (pc *RealPodControl) CreatePod(pod *corev1.Pod) (*corev1.Pod, error) { + if err := pc.client.Create(context.TODO(), pod); err != nil { + return nil, fmt.Errorf("fail to create Pod: %s", err) + } + + return pod, nil +} + +func (pc *RealPodControl) DeletePod(pod *corev1.Pod) error { + return pc.client.Delete(context.TODO(), pod) +} + +func (pc *RealPodControl) UpdatePod(pod *corev1.Pod) error { + return pc.client.Update(context.TODO(), pod) +} + +func (pc *RealPodControl) getPodSetPods(pods []*corev1.Pod, selector *metav1.LabelSelector, owner client.Object) ([]*corev1.Pod, error) { + // Use ControllerRefManager to adopt/orphan as needed. + cm, err := refmanagerutil.NewRefManager(pc.client, selector, owner, pc.scheme) + if err != nil { + return nil, fmt.Errorf("fail to create ref manager: %s", err) + } + + var candidates = make([]client.Object, len(pods)) + for i, pod := range pods { + candidates[i] = pod + } + + claims, err := cm.ClaimOwned(candidates) + if err != nil { + return nil, err + } + + claimPods := make([]*corev1.Pod, len(claims)) + for i, mt := range claims { + claimPods[i] = mt.(*corev1.Pod) + } + + return claimPods, nil +} + +func FilterOutInactivePod(pods []corev1.Pod) []*corev1.Pod { + var filteredPod []*corev1.Pod + + for i := range pods { + if IsPodInactive(&pods[i]) { + continue + } + + filteredPod = append(filteredPod, &pods[i]) + } + return filteredPod +} + +func IsPodInactive(pod *corev1.Pod) bool { + return pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed +} diff --git a/pkg/controllers/collaset/revision.go b/pkg/controllers/collaset/revision.go new file mode 100644 index 00000000..6428f956 --- /dev/null +++ b/pkg/controllers/collaset/revision.go @@ -0,0 +1,95 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collaset + +import ( + "encoding/json" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + appsalphav1 "kusionstack.io/kafed/apis/apps/v1alpha1" +) + +func getCollaSetPatch(ds *appsalphav1.CollaSet) ([]byte, error) { + dsBytes, err := json.Marshal(ds) + if err != nil { + return nil, err + } + var raw map[string]interface{} + err = json.Unmarshal(dsBytes, &raw) + if err != nil { + return nil, err + } + objCopy := make(map[string]interface{}) + specCopy := make(map[string]interface{}) + + // Create a patch of the DaemonSet that replaces spec.template + spec := raw["spec"].(map[string]interface{}) + template := spec["template"].(map[string]interface{}) + template["$patch"] = "replace" + specCopy["template"] = template + + if _, exist := spec["volumeClaimTemplates"]; exist { + specCopy["volumeClaimTemplates"] = spec["volumeClaimTemplates"] + } + + objCopy["spec"] = specCopy + patch, err := json.Marshal(objCopy) + return patch, err +} + +type revisionOwnerAdapter struct { +} + +func (roa *revisionOwnerAdapter) GetSelector(obj metav1.Object) *metav1.LabelSelector { + ips, _ := obj.(*appsalphav1.CollaSet) + return ips.Spec.Selector +} + +func (roa *revisionOwnerAdapter) GetCollisionCount(obj metav1.Object) *int32 { + ips, _ := obj.(*appsalphav1.CollaSet) + return ips.Status.CollisionCount +} + +func (roa *revisionOwnerAdapter) GetHistoryLimit(obj metav1.Object) int32 { + ips, _ := obj.(*appsalphav1.CollaSet) + return ips.Spec.HistoryLimit +} + +func (roa *revisionOwnerAdapter) GetPatch(obj metav1.Object) ([]byte, error) { + cs, _ := obj.(*appsalphav1.CollaSet) + return getCollaSetPatch(cs) +} + +func (roa *revisionOwnerAdapter) GetSelectorLabels(obj metav1.Object) map[string]string { + ips, _ := obj.(*appsalphav1.CollaSet) + labels := map[string]string{} + for k, v := range ips.Spec.Template.Labels { + labels[k] = v + } + + return labels +} + +func (roa *revisionOwnerAdapter) GetCurrentRevision(obj metav1.Object) string { + ips, _ := obj.(*appsalphav1.CollaSet) + return ips.Status.CurrentRevision +} + +func (roa *revisionOwnerAdapter) IsInUsed(_ metav1.Object, _ string) bool { + return false +} diff --git a/pkg/controllers/collaset/synccontrol/scale.go b/pkg/controllers/collaset/synccontrol/scale.go new file mode 100644 index 00000000..6d5c7e8f --- /dev/null +++ b/pkg/controllers/collaset/synccontrol/scale.go @@ -0,0 +1,56 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package synccontrol + +import ( + "sort" + + collasetutils "kusionstack.io/kafed/pkg/controllers/collaset/utils" + controllerutils "kusionstack.io/kafed/pkg/controllers/utils" + "kusionstack.io/kafed/pkg/controllers/utils/podopslifecycle" +) + +func getPodsToDelete(filteredPods []*collasetutils.PodWrapper, diff int) []*collasetutils.PodWrapper { + sort.Sort(ActivePodsForDeletion(filteredPods)) + start := len(filteredPods) - diff + if start < 0 { + start = 0 + } + return filteredPods[start:] +} + +type ActivePodsForDeletion []*collasetutils.PodWrapper + +func (s ActivePodsForDeletion) Len() int { return len(s) } +func (s ActivePodsForDeletion) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func (s ActivePodsForDeletion) Less(i, j int) bool { + l, r := s[i], s[j] + + lDuringScaleIn := podopslifecycle.IsDuringOps(collasetutils.ScaleInOpsLifecycleAdapter, l) + rDuringScaleIn := podopslifecycle.IsDuringOps(collasetutils.ScaleInOpsLifecycleAdapter, r) + + if lDuringScaleIn && !rDuringScaleIn { + return true + } + + if !lDuringScaleIn && rDuringScaleIn { + return false + } + + return controllerutils.ComparePod(l.Pod, r.Pod) +} diff --git a/pkg/controllers/collaset/synccontrol/sync_control.go b/pkg/controllers/collaset/synccontrol/sync_control.go new file mode 100644 index 00000000..aef54007 --- /dev/null +++ b/pkg/controllers/collaset/synccontrol/sync_control.go @@ -0,0 +1,500 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package synccontrol + +import ( + "fmt" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "kusionstack.io/kafed/apis/apps/v1alpha1" + "kusionstack.io/kafed/pkg/controllers/collaset/podcontext" + "kusionstack.io/kafed/pkg/controllers/collaset/podcontrol" + "kusionstack.io/kafed/pkg/controllers/collaset/utils" + collasetutils "kusionstack.io/kafed/pkg/controllers/collaset/utils" + controllerutils "kusionstack.io/kafed/pkg/controllers/utils" + "kusionstack.io/kafed/pkg/controllers/utils/expectations" + "kusionstack.io/kafed/pkg/controllers/utils/podopslifecycle" +) + +const ( + ScaleInContextDataKey = "ScaleIn" +) + +type Interface interface { + SyncPods(instance *appsv1alpha1.CollaSet, updatedRevision *appsv1.ControllerRevision, newStatus *appsv1alpha1.CollaSetStatus) (bool, []*collasetutils.PodWrapper, map[int]*appsv1alpha1.ContextDetail, error) + Scale(instance *appsv1alpha1.CollaSet, filteredPods []*collasetutils.PodWrapper, revisions []*appsv1.ControllerRevision, updatedRevision *appsv1.ControllerRevision, ownedIDs map[int]*appsv1alpha1.ContextDetail, newStatus *appsv1alpha1.CollaSetStatus) (bool, error) + Update(instance *appsv1alpha1.CollaSet, filteredPods []*collasetutils.PodWrapper, revisions []*appsv1.ControllerRevision, updatedRevision *appsv1.ControllerRevision, ownedIDs map[int]*appsv1alpha1.ContextDetail, newStatus *appsv1alpha1.CollaSetStatus) (bool, error) +} + +func NewRealSyncControl(client client.Client, podControl podcontrol.Interface, recorder record.EventRecorder) *RealSyncControl { + return &RealSyncControl{ + client: client, + podControl: podControl, + recorder: recorder, + } +} + +type RealSyncControl struct { + client client.Client + podControl podcontrol.Interface + recorder record.EventRecorder +} + +// SyncPods is used to reclaim Pod instance ID +func (sc *RealSyncControl) SyncPods(instance *appsv1alpha1.CollaSet, updatedRevision *appsv1.ControllerRevision, _ *appsv1alpha1.CollaSetStatus) (bool, []*collasetutils.PodWrapper, map[int]*appsv1alpha1.ContextDetail, error) { + filteredPods, err := sc.podControl.GetFilteredPods(instance.Spec.Selector, instance) + if err != nil { + return false, nil, nil, fmt.Errorf("fail to get filtered Pods: %s", err) + } + + // get owned IDs + var ownedIDs map[int]*appsv1alpha1.ContextDetail + if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + ownedIDs, err = podcontext.AllocateID(sc.client, instance, updatedRevision.Name, int(replicasRealValue(instance.Spec.Replicas))) + return err + }); err != nil { + return false, nil, ownedIDs, fmt.Errorf("fail to allocate %d IDs using context when sync Pods: %s", instance.Spec.Replicas, err) + } + + // wrap Pod with more information + var podWrappers []*collasetutils.PodWrapper + + // stateless case + currentIDs := sets.Int{} + idToReclaim := sets.Int{} + for i := range filteredPods { + pod := filteredPods[i] + id, _ := collasetutils.GetPodInstanceID(pod) + if pod.DeletionTimestamp != nil { + // 1. Reclaim ID from Pod which is scaling in and terminating. + if contextDetail, exist := ownedIDs[id]; exist && contextDetail.Contains(ScaleInContextDataKey, "true") { + idToReclaim.Insert(id) + } + + // 2. filter out Pods which are terminating + continue + } + + podWrappers = append(podWrappers, &collasetutils.PodWrapper{ + Pod: pod, + ID: id, + ContextDetail: ownedIDs[id], + }) + + if id >= 0 { + currentIDs.Insert(id) + } + } + + // 3. Reclaim Pod ID which Pod & PVC are all non-existing + for id, contextDetail := range ownedIDs { + if contextDetail.Contains(ScaleInContextDataKey, "true") && !currentIDs.Has(id) { + idToReclaim.Insert(id) + } + } + + needUpdateContext := false + for _, id := range idToReclaim.List() { + needUpdateContext = true + delete(ownedIDs, id) + } + + // TODO stateful case + // 1. only reclaim non-existing Pods' ID. Do not reclaim terminating Pods' ID until these Pods and PVC have been deleted from ETCD + // 2. do not filter out these terminating Pods + + if needUpdateContext { + klog.V(1).Infof("try to update ResourceContext for CollaSet %s/%s when sync", instance.Namespace, instance.Name) + if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + return podcontext.UpdateToPodContext(sc.client, instance, ownedIDs) + }); err != nil { + return false, nil, ownedIDs, fmt.Errorf("fail to update ResourceContext when reclaiming IDs: %s", err) + } + } + + return false, podWrappers, ownedIDs, nil +} + +func (sc *RealSyncControl) Scale(set *appsv1alpha1.CollaSet, podWrappers []*collasetutils.PodWrapper, revisions []*appsv1.ControllerRevision, updatedRevision *appsv1.ControllerRevision, ownedIDs map[int]*appsv1alpha1.ContextDetail, newStatus *appsv1alpha1.CollaSetStatus) (scaled bool, err error) { + diff := int(replicasRealValue(set.Spec.Replicas)) - len(podWrappers) + scaling := false + + if diff > 0 { + // collect instance ID in used from owned Pods + podInstanceIDSet := collasetutils.CollectPodInstanceID(podWrappers) + // find IDs and their contexts which have not been used by owned Pods + availableContext := extractAvailableContexts(diff, ownedIDs, podInstanceIDSet) + + succCount, err := controllerutils.SlowStartBatch(diff, controllerutils.SlowStartInitialBatchSize, false, func(idx int, _ error) error { + availableIDContext := availableContext[idx] + // use revision recorded in Context + revision := updatedRevision + if revisionName, exist := availableIDContext.Data[podcontext.RevisionContextDataKey]; exist && revisionName != "" { + for i := range revisions { + if revisions[i].Name == revisionName { + revision = revisions[i] + break + } + } + } + + // scale out new Pods with updatedRevision + // TODO use cache + pod, err := controllerutils.NewPodFrom(set, metav1.NewControllerRef(set, appsv1alpha1.GroupVersion.WithKind("CollaSet")), revision) + if err != nil { + return fmt.Errorf("fail to new Pod from revision %s: %s", revision.Name, err) + } + newPod := pod.DeepCopy() + // allocate new Pod a instance ID + newPod.Labels[appsv1alpha1.PodInstanceIDLabelKey] = fmt.Sprintf("%d", availableIDContext.ID) + + klog.V(1).Info("try to create Pod with revision %s from CollaSet %s/%s", revision.Name, set.Namespace, set.Name) + if pod, err := sc.podControl.CreatePod(newPod); err == nil { + // add an expectation for this pod creation, before next reconciling + if err := collasetutils.ActiveExpectations.ExpectCreate(set, expectations.Pod, pod.Name); err != nil { + return err + } + } + return err + }) + + sc.recorder.Eventf(set, corev1.EventTypeNormal, "ScaleOut", "scale out %d Pod(s)", succCount) + if err != nil { + collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetScale, err, "ScaleOutFailed", err.Error()) + return succCount > 0, err + } + collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetScale, nil, "ScaleOut", "") + + return succCount > 0, err + } else if diff < 0 { + // chose the pods to scale in + podsToScaleIn := getPodsToDelete(podWrappers, diff*-1) + // filter out Pods need to trigger PodOpsLifecycle + podCh := make(chan *collasetutils.PodWrapper, len(podsToScaleIn)) + for i := range podsToScaleIn { + if podopslifecycle.IsDuringOps(collasetutils.ScaleInOpsLifecycleAdapter, podsToScaleIn[i].Pod) { + continue + } + podCh <- podsToScaleIn[i] + } + + // trigger Pods to enter PodOpsLifecycle + succCount, err := controllerutils.SlowStartBatch(len(podCh), controllerutils.SlowStartInitialBatchSize, false, func(_ int, err error) error { + pod := <-podCh + + // trigger PodOpsLifecycle with scaleIn OperationType + klog.V(1).Infof("try to begin PodOpsLifecycle for scaling in Pod %s/%s in CollaSet %s/%s", pod.Namespace, pod.Name, set.Namespace, set.Name) + if updated, err := podopslifecycle.Begin(sc.client, collasetutils.ScaleInOpsLifecycleAdapter, pod.Pod); err != nil { + return fmt.Errorf("fail to begin PodOpsLifecycle for Scaling in Pod %s/%s: %s", pod.Namespace, pod.Name, err) + } else if updated { + sc.recorder.Eventf(pod.Pod, corev1.EventTypeNormal, "BeginScaleInLifecycle", "succeed to begin PodOpsLifecycle for scaling in") + // add an expectation for this pod creation, before next reconciling + if err := collasetutils.ActiveExpectations.ExpectUpdate(set, expectations.Pod, pod.Name, pod.ResourceVersion); err != nil { + return err + } + } + + return nil + }) + scaling = succCount != 0 + + if err != nil { + collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetScale, err, "ScaleInFailed", err.Error()) + return scaling, err + } else { + collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetScale, nil, "ScaleIn", "") + } + + needUpdateContext := false + for i, podWrapper := range podsToScaleIn { + if !podopslifecycle.AllowOps(collasetutils.ScaleInOpsLifecycleAdapter, podWrapper.Pod) && podWrapper.DeletionTimestamp == nil { + sc.recorder.Eventf(podWrapper.Pod, corev1.EventTypeNormal, "PodScaleInLifecycle", "Pod is not allowed to scale in") + continue + } + + // if Pod is allowed to operate or Pod has already been deleted, promte to delete Pod + if podWrapper.ID >= 0 && !ownedIDs[podWrapper.ID].Contains(ScaleInContextDataKey, "true") { + needUpdateContext = true + ownedIDs[podWrapper.ID].Put(ScaleInContextDataKey, "true") + } + + if podWrapper.DeletionTimestamp != nil { + continue + } + + podCh <- podsToScaleIn[i] + } + + // mark these Pods to scalingIn + if needUpdateContext { + klog.V(1).Infof("try to update ResourceContext for CollaSet %s/%s when scaling in Pod", set.Namespace, set.Name) + err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { + return podcontext.UpdateToPodContext(sc.client, set, ownedIDs) + }) + + if err != nil { + collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetScale, err, "ScaleInFailed", fmt.Sprintf("failed to update Context for scaling in: %s", err)) + return scaling, err + } else { + collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetScale, nil, "ScaleIn", "") + } + } + + // do delete Pod resource + succCount, err = controllerutils.SlowStartBatch(len(podCh), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { + pod := <-podCh + klog.V(1).Infof("try to scale in Pod %s/%s", pod.Namespace, pod.Name) + if err := sc.podControl.DeletePod(pod.Pod); err != nil { + return fmt.Errorf("fail to delete Pod %s/%s when scaling in: %s", pod.Namespace, pod.Name, err) + } + + sc.recorder.Eventf(set, corev1.EventTypeNormal, "PodDeleted", "succeed to scale in Pod %s/%s", pod.Namespace, pod.Name) + if err := collasetutils.ActiveExpectations.ExpectDelete(set, expectations.Pod, pod.Name); err != nil { + return err + } + + // TODO also need to delete PVC from PVC template here + + return nil + }) + scaling := scaling || succCount > 0 + + if succCount > 0 { + sc.recorder.Eventf(set, corev1.EventTypeNormal, "ScaleIn", "scale in %d Pod(s)", succCount) + } + if err != nil { + collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetScale, err, "ScaleInFailed", fmt.Sprintf("fail to delete Pod for scaling in: %s", err)) + return scaling, err + } else { + collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetScale, nil, "ScaleIn", "") + } + + return scaling, err + } + + // reset ContextDetail.ScalingIn, if there are Pods had its PodOpsLifecycle reverted + needUpdatePodContext := false + for _, podWrapper := range podWrappers { + if !podopslifecycle.IsDuringOps(collasetutils.ScaleInOpsLifecycleAdapter, podWrapper) && ownedIDs[podWrapper.ID].Contains(ScaleInContextDataKey, "true") { + needUpdatePodContext = true + ownedIDs[podWrapper.ID].Remove(ScaleInContextDataKey) + } + } + + if needUpdatePodContext { + klog.V(1).Infof("try to update ResourceContext for CollaSet %s/%s after scaling", set.Namespace, set.Name) + if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + return podcontext.UpdateToPodContext(sc.client, set, ownedIDs) + }); err != nil { + return scaling, fmt.Errorf("fail to reset ResourceContext: %s", err) + } + } + + return scaling, nil +} + +func extractAvailableContexts(diff int, ownedIDs map[int]*appsv1alpha1.ContextDetail, podInstanceIDSet map[int]struct{}) []*appsv1alpha1.ContextDetail { + availableContexts := make([]*appsv1alpha1.ContextDetail, diff) + + idx := 0 + for id := range ownedIDs { + if _, inUsed := podInstanceIDSet[id]; inUsed { + continue + } + + availableContexts[idx] = ownedIDs[id] + idx++ + } + + return availableContexts +} + +func (sc *RealSyncControl) Update(set *appsv1alpha1.CollaSet, podWrapers []*collasetutils.PodWrapper, revisions []*appsv1.ControllerRevision, updatedRevision *appsv1.ControllerRevision, ownedIDs map[int]*appsv1alpha1.ContextDetail, newStatus *appsv1alpha1.CollaSetStatus) (bool, error) { + // 1. scan and analysis pods update info + podUpdateInfos := attachPodUpdateInfo(podWrapers, revisions, updatedRevision) + + // 2. decide Pod update candidates + podToUpdate := decidePodToUpdate(set, podUpdateInfos) + + // 3. prepare Pods to begin PodOpsLifecycle + podCh := make(chan *PodUpdateInfo, len(podToUpdate)) + for _, podInfo := range podToUpdate { + if podInfo.IsUpdatedRevision { + continue + } + + if podopslifecycle.IsDuringOps(utils.UpdateOpsLifecycleAdapter, podInfo) { + continue + } + + podCh <- podInfo + } + + // 4. begin podOpsLifecycle parallel + updater := newPodUpdater(set) + updating := false + succCount, err := controllerutils.SlowStartBatch(len(podCh), controllerutils.SlowStartInitialBatchSize, false, func(_ int, err error) error { + podInfo := <-podCh + + klog.V(1).Infof("try to begin PodOpsLifecycle for updating Pod %s/%s of CollaSet %s/%s", podInfo.Namespace, podInfo.Name, set.Namespace, set.Name) + if updated, err := podopslifecycle.Begin(sc.client, utils.UpdateOpsLifecycleAdapter, podInfo.Pod); err != nil { + return fmt.Errorf("fail to begin PodOpsLifecycle for updating Pod %s/%s: %s", podInfo.Namespace, podInfo.Name, err) + } else if updated { + // add an expectation for this pod update, before next reconciling + if err := collasetutils.ActiveExpectations.ExpectUpdate(set, expectations.Pod, podInfo.Name, podInfo.ResourceVersion); err != nil { + return err + } + } + + return nil + }) + + updating = updating || succCount > 0 + if err != nil { + collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetUpdate, err, "UpdateFailed", err.Error()) + return updating, err + } else { + collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetUpdate, nil, "Updated", "") + } + + needUpdateContext := false + for i := range podToUpdate { + podInfo := podToUpdate[i] + if !podopslifecycle.AllowOps(utils.UpdateOpsLifecycleAdapter, podInfo) { + sc.recorder.Eventf(podInfo, corev1.EventTypeNormal, "PodUpdateLifecycle", "Pod is not allowed to update") + continue + } + + if !ownedIDs[podInfo.ID].Contains(podcontext.RevisionContextDataKey, updatedRevision.Name) { + needUpdateContext = true + ownedIDs[podInfo.ID].Put(podcontext.RevisionContextDataKey, updatedRevision.Name) + } + + if podInfo.IsUpdatedRevision { + continue + } + + // if Pod has not been updated, update it. + podCh <- podToUpdate[i] + } + + // 5. mark Pod to use updated revision before updating it. + if needUpdateContext { + klog.V(1).Infof("try to update ResourceContext for CollaSet %s/%s", set.Namespace, set.Name) + err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { + return podcontext.UpdateToPodContext(sc.client, set, ownedIDs) + }) + + if err != nil { + collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetScale, err, "UpdateFailed", fmt.Sprintf("fail to update Context for updating: %s", err)) + return updating, err + } else { + collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetScale, nil, "UpdateFailed", "") + } + } + + // 6. update Pod + succCount, err = controllerutils.SlowStartBatch(len(podCh), controllerutils.SlowStartInitialBatchSize, false, func(_ int, _ error) error { + podInfo := <-podCh + + // analyse Pod to get update information + inPlaceSupport, onlyMetadataChanged, updatedPod, err := updater.AnalyseAndGetUpdatedPod(set, updatedRevision, podInfo) + if err != nil { + return fmt.Errorf("fail to analyse pod %s/%s in-place update support: %s", podInfo.Namespace, podInfo.Name, err) + } + + klog.V(1).Infof("Pod %s/%s update operation from revision %s to revision %s, is [%t] in-place update supported and [%t] only has metadata changed.", + podInfo.Namespace, podInfo.Name, podInfo.CurrentRevision.Name, updatedRevision.Namespace, inPlaceSupport, onlyMetadataChanged) + if onlyMetadataChanged || inPlaceSupport { + // 6.1 if pod template changes only include metadata or support in-place update, just apply these changes to pod directly + if err = sc.podControl.UpdatePod(updatedPod); err != nil { + return fmt.Errorf("fail to update Pod %s/%s when updating by in-place: %s", podInfo.Namespace, podInfo.Name, err) + } else { + podInfo.Pod = updatedPod + sc.recorder.Eventf(podInfo.Pod, corev1.EventTypeNormal, "UpdatePod", "succeed to update Pod %s/%s to from revision %s to revision %s by in-place", podInfo.Namespace, podInfo.Name, podInfo.CurrentRevision.Name, updatedRevision.Name) + if err := collasetutils.ActiveExpectations.ExpectUpdate(set, expectations.Pod, podInfo.Name, updatedPod.ResourceVersion); err != nil { + return err + } + } + } else { + // 6.2 if pod has changes not in-place supported, recreate it + if err = sc.podControl.DeletePod(podInfo.Pod); err != nil { + return fmt.Errorf("fail to delete Pod %s/%s when updating by recreate: %s", podInfo.Namespace, podInfo.Name, err) + } else { + sc.recorder.Eventf(podInfo.Pod, corev1.EventTypeNormal, "UpdatePod", "succeed to update Pod %s/%s to from revision %s to revision %s by recreate", podInfo.Namespace, podInfo.Name, podInfo.CurrentRevision.Name, updatedRevision.Name) + if err := collasetutils.ActiveExpectations.ExpectDelete(set, expectations.Pod, podInfo.Name); err != nil { + return err + } + } + } + + return nil + }) + + updating = updating || succCount > 0 + if err != nil { + collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetUpdate, err, "UpdateFailed", err.Error()) + return updating, err + } else { + collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetUpdate, nil, "Updated", "") + } + + // try to finish all Pods'PodOpsLifecycle if its update is finished. + succCount, err = controllerutils.SlowStartBatch(len(podWrapers), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { + podInfo := podWrapers[i] + + // check Pod update is finished or not + finished, msg, err := updater.GetPodUpdateFinishStatus(podInfo.Pod) + if err != nil { + return fmt.Errorf("fail to get pod %s/%s update finished: %s", podInfo.Namespace, podInfo.Name, err) + } + + if finished { + klog.V(1).Infof("try to finish update PodOpsLifecycle for Pod %s/%s", podInfo.Namespace, podInfo.Name) + if updated, err := podopslifecycle.Finish(sc.client, utils.UpdateOpsLifecycleAdapter, podInfo.Pod); err != nil { + return fmt.Errorf("fail to finish PodOpsLifecycle for updating Pod %s/%s: %s", podInfo.Namespace, podInfo.Name, err) + } else if updated { + // add an expectation for this pod update, before next reconciling + if err := collasetutils.ActiveExpectations.ExpectUpdate(set, expectations.Pod, podInfo.Name, podInfo.ResourceVersion); err != nil { + return err + } + sc.recorder.Eventf(podInfo.Pod, corev1.EventTypeNormal, "UpdateReady", "pod %s/%s update finished", podInfo.Namespace, podInfo.Name) + } + } else { + sc.recorder.Eventf(podInfo.Pod, corev1.EventTypeNormal, "WaitingUpdateReady", "waiting for pod %s/%s to update finished: %s", podInfo.Namespace, podInfo.Name, msg) + } + + return nil + }) + + return updating || succCount > 0, err +} + +func replicasRealValue(replcias *int32) int32 { + if replcias == nil { + return 0 + } + + return *replcias +} diff --git a/pkg/controllers/collaset/synccontrol/update.go b/pkg/controllers/collaset/synccontrol/update.go new file mode 100644 index 00000000..18b2ac8a --- /dev/null +++ b/pkg/controllers/collaset/synccontrol/update.go @@ -0,0 +1,352 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package synccontrol + +import ( + "encoding/json" + "fmt" + "sort" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + appsv1alpha1 "kusionstack.io/kafed/apis/apps/v1alpha1" + "kusionstack.io/kafed/pkg/controllers/collaset/utils" + collasetutils "kusionstack.io/kafed/pkg/controllers/collaset/utils" + controllerutils "kusionstack.io/kafed/pkg/controllers/utils" + "kusionstack.io/kafed/pkg/controllers/utils/podopslifecycle" +) + +type PodUpdateInfo struct { + *utils.PodWrapper + + // indicate if this pod has up-to-date revision from its owner, like CollaSet + IsUpdatedRevision bool + // carry the pod's current revision + CurrentRevision *appsv1.ControllerRevision + + // indicates the PodOpsLifecycle is started. + isDuringOps bool +} + +func attachPodUpdateInfo(pods []*collasetutils.PodWrapper, revisions []*appsv1.ControllerRevision, updatedRevision *appsv1.ControllerRevision) []*PodUpdateInfo { + podUpdateInfoList := make([]*PodUpdateInfo, len(pods)) + + for i, pod := range pods { + updateInfo := &PodUpdateInfo{PodWrapper: pod} + + // decide this pod current revision, or nil if not indicated + if pod.Labels != nil { + currentRevisionName, exist := pod.Labels[appsv1.ControllerRevisionHashLabelKey] + if exist { + if currentRevisionName == updatedRevision.Name { + updateInfo.IsUpdatedRevision = true + updateInfo.CurrentRevision = updatedRevision + } else { + updateInfo.IsUpdatedRevision = false + for _, rv := range revisions { + if currentRevisionName == rv.Name { + updateInfo.CurrentRevision = rv + } + } + } + } + } + + // decide whether the PodOpsLifecycle is during ops or not + updateInfo.isDuringOps = podopslifecycle.IsDuringOps(utils.UpdateOpsLifecycleAdapter, pod) + + podUpdateInfoList[i] = updateInfo + } + + return podUpdateInfoList +} + +func decidePodToUpdate(cls *appsv1alpha1.CollaSet, podInfos []*PodUpdateInfo) []*PodUpdateInfo { + if cls.Spec.UpdateStrategy.RollingUpdate != nil && cls.Spec.UpdateStrategy.RollingUpdate.ByLabel != nil { + return decidePodToUpdateByLabel(cls, podInfos) + } + + return decidePodToUpdateByPartition(cls, podInfos) +} + +func decidePodToUpdateByLabel(_ *appsv1alpha1.CollaSet, podInfos []*PodUpdateInfo) (podToUpdate []*PodUpdateInfo) { + for i := range podInfos { + if _, exist := podInfos[i].Labels[appsv1alpha1.CollaSetUpdateIndicateLabelKey]; exist { + podToUpdate = append(podToUpdate, podInfos[i]) + } + } + + return podToUpdate +} + +func decidePodToUpdateByPartition(cls *appsv1alpha1.CollaSet, podInfos []*PodUpdateInfo) (podToUpdate []*PodUpdateInfo) { + if cls.Spec.UpdateStrategy.RollingUpdate == nil || + cls.Spec.UpdateStrategy.RollingUpdate.ByPartition.Partition == nil { + return podInfos + } + + ordered := orderByDefault(podInfos) + sort.Sort(ordered) + + partition := int(*cls.Spec.UpdateStrategy.RollingUpdate.ByPartition.Partition) + return podInfos[:partition] +} + +type orderByDefault []*PodUpdateInfo + +func (o orderByDefault) Len() int { + return len(o) +} + +func (o orderByDefault) Swap(i, j int) { o[i], o[j] = o[j], o[i] } + +func (o orderByDefault) Less(i, j int) bool { + l, r := o[i], o[j] + if l.IsUpdatedRevision && !r.IsUpdatedRevision { + return true + } + + if !l.IsUpdatedRevision && r.IsUpdatedRevision { + return false + } + + if l.isDuringOps && !r.isDuringOps { + return true + } + + if !l.isDuringOps && r.isDuringOps { + return false + } + + return controllerutils.ComparePod(l.Pod, r.Pod) +} + +type PodUpdater interface { + AnalyseAndGetUpdatedPod(cls *appsv1alpha1.CollaSet, revision *appsv1.ControllerRevision, podUpdateInfo *PodUpdateInfo) (inPlaceUpdateSupport bool, onlyMetadataChanged bool, updatedPod *corev1.Pod, err error) + GetPodUpdateFinishStatus(pod *corev1.Pod) (bool, string, error) +} + +func newPodUpdater(cls *appsv1alpha1.CollaSet) PodUpdater { + switch cls.Spec.UpdateStrategy.PodUpdatePolicy { + case appsv1alpha1.CollaSetRecreatePodUpdateStrategyType: + return &RecreatePodUpdater{} + case appsv1alpha1.CollaSetInPlaceOnlyPodUpdateStrategyType: + // In case of using native K8s, Pod is only allowed to update with container image, so InPlaceOnly policy is + // implemented with InPlaceIfPossible policy as default for compatibility. + return &InPlaceIfPossibleUpdater{} + default: + return &InPlaceIfPossibleUpdater{} + } +} + +type PodStatus struct { + ContainerStates map[string]*ContainerStatus `json:"containerStates,omitempty"` +} + +type ContainerStatus struct { + LatestImage string `json:"latestImage,omitempty"` + LastImageID string `json:"lastImageID,omitempty"` +} + +type InPlaceIfPossibleUpdater struct { +} + +func (u *InPlaceIfPossibleUpdater) AnalyseAndGetUpdatedPod(cls *appsv1alpha1.CollaSet, updatedRevision *appsv1.ControllerRevision, podUpdateInfo *PodUpdateInfo) (inPlaceUpdateSupport bool, onlyMetadataChanged bool, updatedPod *corev1.Pod, err error) { + // 1. build pod from current and updated revision + ownerRef := metav1.NewControllerRef(cls, appsv1alpha1.GroupVersion.WithKind("CollaSet")) + // TODO: use cache + currentPod, err := controllerutils.NewPodFrom(cls, ownerRef, podUpdateInfo.CurrentRevision) + if err != nil { + return false, false, nil, fmt.Errorf("fail to build Pod from current revision %s: %s", podUpdateInfo.CurrentRevision.Name, err) + } + + // TODO: use cache + updatedPod, err = controllerutils.NewPodFrom(cls, ownerRef, updatedRevision) + if err != nil { + return false, false, nil, fmt.Errorf("fail to build Pod from updated revision %s: %s", updatedRevision.Name, err) + } + + // 2. compare current and updated pods. Only pod image and metadata are supported to update in-place + // TODO: use cache + inPlaceUpdateSupport, onlyMetadataChanged = u.diffPod(currentPod, updatedPod) + // 2.1 if pod has changes more than metadata and image + if !inPlaceUpdateSupport { + return false, onlyMetadataChanged, nil, nil + } + + inPlaceUpdateSupport = true + updatedPod, err = controllerutils.PatchToPod(currentPod, updatedPod, podUpdateInfo.Pod) + + if onlyMetadataChanged { + if updatedPod.Annotations != nil { + delete(updatedPod.Annotations, appsv1alpha1.LastPodStatusAnnotationKey) + } + } else { + containerCurrentStatusMapping := map[string]*corev1.ContainerStatus{} + for i := range podUpdateInfo.Status.ContainerStatuses { + status := podUpdateInfo.Status.ContainerStatuses[i] + containerCurrentStatusMapping[status.Name] = &status + } + + podStatus := &PodStatus{ContainerStates: map[string]*ContainerStatus{}} + for _, container := range updatedPod.Spec.Containers { + podStatus.ContainerStates[container.Name] = &ContainerStatus{ + // store image of each container in updated Pod + LatestImage: container.Image, + } + + containerCurrentStatus, exist := containerCurrentStatusMapping[container.Name] + if !exist { + continue + } + + // store image ID of each container in current Pod + podStatus.ContainerStates[container.Name].LastImageID = containerCurrentStatus.ImageID + } + + podStatusStr, err := json.Marshal(podStatus) + if err != nil { + return inPlaceUpdateSupport, onlyMetadataChanged, updatedPod, err + } + + if updatedPod.Annotations == nil { + updatedPod.Annotations = map[string]string{} + } + updatedPod.Annotations[appsv1alpha1.LastPodStatusAnnotationKey] = string(podStatusStr) + } + + return +} + +func (u *InPlaceIfPossibleUpdater) diffPod(currentPod, updatedPod *corev1.Pod) (inPlaceSetUpdateSupport bool, onlyMetadataChanged bool) { + if len(currentPod.Spec.Containers) != len(updatedPod.Spec.Containers) { + return false, false + } + + currentPod = currentPod.DeepCopy() + // sync metadata + currentPod.ObjectMeta = updatedPod.ObjectMeta + + // sync image + imageChanged := false + for i := range currentPod.Spec.Containers { + if currentPod.Spec.Containers[i].Image != updatedPod.Spec.Containers[i].Image { + imageChanged = true + currentPod.Spec.Containers[i].Image = updatedPod.Spec.Containers[i].Image + } + } + + if !equality.Semantic.DeepEqual(currentPod, updatedPod) { + return false, false + } + + if !imageChanged { + return true, true + } + + return true, false +} + +func (u *InPlaceIfPossibleUpdater) GetPodUpdateFinishStatus(pod *corev1.Pod) (finished bool, msg string, err error) { + if pod.Status.ContainerStatuses == nil { + return false, "no container status", nil + } + + if pod.Spec.Containers == nil { + return false, "no container spec", nil + } + + if len(pod.Spec.Containers) != len(pod.Status.ContainerStatuses) { + return false, "container status number does not match", nil + } + + if pod.Annotations == nil { + return true, "no annotations for last container status", nil + } + + podLastState := &PodStatus{} + if lastStateJson, exist := pod.Annotations[appsv1alpha1.LastPodStatusAnnotationKey]; !exist { + return true, "no pod last state annotation", nil + } else if err := json.Unmarshal([]byte(lastStateJson), podLastState); err != nil { + msg := fmt.Sprintf("malformat pod last state annotation [%s]: %s", lastStateJson, err) + return false, msg, fmt.Errorf(msg) + } + + if podLastState.ContainerStates == nil { + return true, "empty last container state recorded", nil + } + + imageMapping := map[string]string{} + for _, containerSpec := range pod.Spec.Containers { + imageMapping[containerSpec.Name] = containerSpec.Image + } + + imageIdMapping := map[string]string{} + for _, containerStatus := range pod.Status.ContainerStatuses { + imageIdMapping[containerStatus.Name] = containerStatus.ImageID + } + + for containerName, lastContaienrState := range podLastState.ContainerStates { + latestImage := lastContaienrState.LatestImage + lastImageId := lastContaienrState.LastImageID + + if currentImage, exist := imageMapping[containerName]; !exist { + // If no this container image recorded, ignore this container. + continue + } else if currentImage != latestImage { + // If container image in pod spec has changed, ignore this container. + continue + } + + if currentImageId, exist := imageIdMapping[containerName]; !exist { + // If no this container image id recorded, ignore this container. + continue + } else if currentImageId == lastImageId { + // No image id changed means the pod in-place update has not finished by kubelet. + return false, fmt.Sprintf("container has %s not been updated: last image id %s, current image id %s", containerName, lastImageId, currentImageId), nil + } + } + + return true, "", nil +} + +// TODO +type InPlaceOnlyPodUpdater struct { +} + +func (u *InPlaceOnlyPodUpdater) AnalyseAndGetUpdatedPod(_ *appsv1alpha1.CollaSet, _ *appsv1.ControllerRevision, _ *PodUpdateInfo) (inPlaceUpdateSupport bool, onlyMetadataChanged bool, updatedPod *corev1.Pod, err error) { + + return +} + +func (u *InPlaceOnlyPodUpdater) GetPodUpdateFinishStatus(_ *corev1.Pod) (finished bool, msg string, err error) { + return +} + +type RecreatePodUpdater struct { +} + +func (u *RecreatePodUpdater) AnalyseAndGetUpdatedPod(_ *appsv1alpha1.CollaSet, _ *appsv1.ControllerRevision, _ *PodUpdateInfo) (inPlaceUpdateSupport bool, onlyMetadataChanged bool, updatedPod *corev1.Pod, err error) { + return false, false, nil, nil +} + +func (u *RecreatePodUpdater) GetPodUpdateFinishStatus(_ *corev1.Pod) (finished bool, msg string, err error) { + // Recreate policy alway treat Pod as update finished + return true, "", nil +} diff --git a/pkg/controllers/collaset/utils/condition.go b/pkg/controllers/collaset/utils/condition.go new file mode 100644 index 00000000..4a81c375 --- /dev/null +++ b/pkg/controllers/collaset/utils/condition.go @@ -0,0 +1,98 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + appsv1alpha1 "kusionstack.io/kafed/apis/apps/v1alpha1" +) + +const ConditionUpdatePeriodBackOff = 30 * time.Second + +func AddOrUpdateCondition(status *appsv1alpha1.CollaSetStatus, conditionType appsv1alpha1.CollaSetConditionType, err error, reason, message string) { + condStatus := corev1.ConditionTrue + if err != nil { + condStatus = corev1.ConditionFalse + } + + existCond := GetCondition(status, conditionType) + if existCond != nil { + if existCond.Reason == reason && + existCond.Message == message && + existCond.Status == condStatus { + now := metav1.Now() + if now.Sub(existCond.LastTransitionTime.Time) < ConditionUpdatePeriodBackOff { + return + } + } + } + + cond := NewCondition(conditionType, condStatus, reason, message) + SetCondition(status, cond) +} + +func NewCondition(condType appsv1alpha1.CollaSetConditionType, status corev1.ConditionStatus, reason, msg string) *appsv1alpha1.CollaSetCondition { + return &appsv1alpha1.CollaSetCondition{ + Type: condType, + Status: status, + LastTransitionTime: metav1.Now(), + Reason: reason, + Message: msg, + } +} + +// GetCondition returns a inplace set condition with the provided type if it exists. +func GetCondition(status *appsv1alpha1.CollaSetStatus, condType appsv1alpha1.CollaSetConditionType) *appsv1alpha1.CollaSetCondition { + for _, c := range status.Conditions { + if c.Type == condType { + return &c + } + } + return nil +} + +// SetCondition adds/replaces the given condition in the replicaset status. If the condition that we +// are about to add already exists and has the same status and reason then we are not going to update. +func SetCondition(status *appsv1alpha1.CollaSetStatus, condition *appsv1alpha1.CollaSetCondition) { + currentCond := GetCondition(status, condition.Type) + if currentCond != nil && currentCond.Status == condition.Status && currentCond.Reason == condition.Reason && currentCond.LastTransitionTime == condition.LastTransitionTime { + return + } + newConditions := filterOutCondition(status.Conditions, condition.Type) + status.Conditions = append(newConditions, *condition) +} + +// RemoveCondition removes the condition with the provided type from the replicaset status. +func RemoveCondition(status *appsv1alpha1.CollaSetStatus, condType appsv1alpha1.CollaSetConditionType) { + status.Conditions = filterOutCondition(status.Conditions, condType) +} + +// filterOutCondition returns a new slice of replicaset conditions without conditions with the provided type. +func filterOutCondition(conditions []appsv1alpha1.CollaSetCondition, condType appsv1alpha1.CollaSetConditionType) []appsv1alpha1.CollaSetCondition { + var newConditions []appsv1alpha1.CollaSetCondition + for _, c := range conditions { + if c.Type == condType { + continue + } + newConditions = append(newConditions, c) + } + return newConditions +} diff --git a/pkg/controllers/collaset/utils/expectation.go b/pkg/controllers/collaset/utils/expectation.go new file mode 100644 index 00000000..37d98825 --- /dev/null +++ b/pkg/controllers/collaset/utils/expectation.go @@ -0,0 +1,32 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + "kusionstack.io/kafed/pkg/controllers/utils/expectations" +) + +var ( + // ActiveExpectations is used to check the cache in informer is updated, before reconciling. + ActiveExpectations *expectations.ActiveExpectations +) + +func InitExpectations(c client.Client) { + ActiveExpectations = expectations.NewActiveExpectations(c) +} diff --git a/pkg/controllers/collaset/utils/lifecycle_adapter.go b/pkg/controllers/collaset/utils/lifecycle_adapter.go new file mode 100644 index 00000000..6da6f82d --- /dev/null +++ b/pkg/controllers/collaset/utils/lifecycle_adapter.go @@ -0,0 +1,100 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "kusionstack.io/kafed/apis/apps/v1alpha1" + "kusionstack.io/kafed/pkg/controllers/utils/podopslifecycle" +) + +var ( + UpdateOpsLifecycleAdapter = &CollaSetUpdateOpsLifecycleAdapter{} + ScaleInOpsLifecycleAdapter = &CollaSetScaleInOpsLifecycleAdapter{} +) + +// CollaSetUpdateOpsLifecycleAdapter tells PodOpsLifecycle the basic workload update ops info +type CollaSetUpdateOpsLifecycleAdapter struct { +} + +// GetID indicates ID of one PodOpsLifecycle +func (a *CollaSetUpdateOpsLifecycleAdapter) GetID() string { + return "collaset" +} + +// GetType indicates type for an Operator +func (a *CollaSetUpdateOpsLifecycleAdapter) GetType() podopslifecycle.OperationType { + return podopslifecycle.OpsLifecycleTypeUpdate +} + +// AllowMultiType indicates whether multiple IDs which have the same Type are allowed +func (a *CollaSetUpdateOpsLifecycleAdapter) AllowMultiType() bool { + return true +} + +// WhenBegin will be executed when begin a lifecycle +func (a *CollaSetUpdateOpsLifecycleAdapter) WhenBegin(_ client.Object) (bool, error) { + return false, nil +} + +// WhenFinish will be executed when finish a lifecycle +func (a *CollaSetUpdateOpsLifecycleAdapter) WhenFinish(pod client.Object) (bool, error) { + needUpdated := false + if _, exist := pod.GetLabels()[appsv1alpha1.CollaSetUpdateIndicateLabelKey]; exist { + delete(pod.GetLabels(), appsv1alpha1.CollaSetUpdateIndicateLabelKey) + needUpdated = true + } + + if pod.GetAnnotations() != nil { + if _, exist := pod.GetAnnotations()[appsv1alpha1.LastPodStatusAnnotationKey]; exist { + delete(pod.GetAnnotations(), appsv1alpha1.LastPodStatusAnnotationKey) + needUpdated = true + } + } + + return needUpdated, nil +} + +// CollaSetScaleInOpsLifecycleAdapter tells PodOpsLifecycle the basic workload scaling in ops info +type CollaSetScaleInOpsLifecycleAdapter struct { +} + +// GetID indicates ID of one PodOpsLifecycle +func (a *CollaSetScaleInOpsLifecycleAdapter) GetID() string { + return "collaset" +} + +// GetType indicates type for an Operator +func (a *CollaSetScaleInOpsLifecycleAdapter) GetType() podopslifecycle.OperationType { + return podopslifecycle.OpsLifecycleTypeScaleIn +} + +// AllowMultiType indicates whether multiple IDs which have the same Type are allowed +func (a *CollaSetScaleInOpsLifecycleAdapter) AllowMultiType() bool { + return true +} + +// WhenBegin will be executed when begin a lifecycle +func (a *CollaSetScaleInOpsLifecycleAdapter) WhenBegin(pod client.Object) (bool, error) { + return false, nil +} + +// WhenFinish will be executed when finish a lifecycle +func (a *CollaSetScaleInOpsLifecycleAdapter) WhenFinish(_ client.Object) (bool, error) { + return false, nil +} diff --git a/pkg/controllers/collaset/utils/pod.go b/pkg/controllers/collaset/utils/pod.go new file mode 100644 index 00000000..fecf585b --- /dev/null +++ b/pkg/controllers/collaset/utils/pod.go @@ -0,0 +1,59 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "fmt" + corev1 "k8s.io/api/core/v1" + "strconv" + + appsv1alpha1 "kusionstack.io/kafed/apis/apps/v1alpha1" +) + +type PodWrapper struct { + *corev1.Pod + ID int + ContextDetail *appsv1alpha1.ContextDetail +} + +func CollectPodInstanceID(pods []*PodWrapper) map[int]struct{} { + podInstanceIDSet := map[int]struct{}{} + for _, pod := range pods { + podInstanceIDSet[pod.ID] = struct{}{} + } + + return podInstanceIDSet +} + +func GetPodInstanceID(pod *corev1.Pod) (int, error) { + if pod.Labels == nil { + return -1, fmt.Errorf("no labels found for instance ID") + } + + val, exist := pod.Labels[appsv1alpha1.PodInstanceIDLabelKey] + if !exist { + return -1, fmt.Errorf("failed to find instance ID label %s", appsv1alpha1.PodInstanceIDLabelKey) + } + + id, err := strconv.ParseInt(val, 10, 32) + if err != nil { + // ignore invalid pod instance ID + return -1, fmt.Errorf("failed to parse instance ID with value %s: %s", val, err) + } + + return int(id), nil +} diff --git a/pkg/controllers/poddeletion/expectation.go b/pkg/controllers/poddeletion/expectation.go new file mode 100644 index 00000000..587af089 --- /dev/null +++ b/pkg/controllers/poddeletion/expectation.go @@ -0,0 +1,32 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package poddeletion + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + "kusionstack.io/kafed/pkg/controllers/utils/expectations" +) + +var ( + // activeExpectations is used to check the cache in informer is updated, before reconciling. + activeExpectations *expectations.ActiveExpectations +) + +func InitExpectations(c client.Client) { + activeExpectations = expectations.NewActiveExpectations(c) +} diff --git a/pkg/controllers/poddeletion/lifecycle_adapter.go b/pkg/controllers/poddeletion/lifecycle_adapter.go new file mode 100644 index 00000000..ec900836 --- /dev/null +++ b/pkg/controllers/poddeletion/lifecycle_adapter.go @@ -0,0 +1,57 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package poddeletion + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + "kusionstack.io/kafed/pkg/controllers/utils/podopslifecycle" +) + +var ( + OpsLifecycleAdapter = &PodDeleteOpsLifecycleAdapter{} +) + +// PodDeleteOpsLifecycleAdapter tells PodOpsLifecycle the Pod deletion ops info +type PodDeleteOpsLifecycleAdapter struct { +} + +// GetID indicates ID of one PodOpsLifecycle +func (a *PodDeleteOpsLifecycleAdapter) GetID() string { + return "pod-delete" +} + +// GetType indicates type for an Operator +func (a *PodDeleteOpsLifecycleAdapter) GetType() podopslifecycle.OperationType { + return podopslifecycle.OpsLifecycleTypeDelete +} + +// AllowMultiType indicates whether multiple IDs which have the same Type are allowed +func (a *PodDeleteOpsLifecycleAdapter) AllowMultiType() bool { + return true +} + +// WhenBegin will be executed when begin a lifecycle +func (a *PodDeleteOpsLifecycleAdapter) WhenBegin(_ client.Object) (bool, error) { + return false, nil +} + +// WhenFinish will be executed when finish a lifecycle +func (a *PodDeleteOpsLifecycleAdapter) WhenFinish(_ client.Object) (bool, error) { + + return false, nil +} diff --git a/pkg/controllers/poddeletion/poddeletion_controller.go b/pkg/controllers/poddeletion/poddeletion_controller.go new file mode 100644 index 00000000..5d5fd2e8 --- /dev/null +++ b/pkg/controllers/poddeletion/poddeletion_controller.go @@ -0,0 +1,134 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package poddeletion + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "kusionstack.io/kafed/pkg/controllers/utils/expectations" + "kusionstack.io/kafed/pkg/controllers/utils/podopslifecycle" +) + +const ( + controllerName = "poddeletion-controller" +) + +// PodDeletionReconciler reconciles and reclaims a Pod object +type PodDeletionReconciler struct { + client.Client + + recorder record.EventRecorder +} + +func Add(mgr ctrl.Manager) error { + return AddToMgr(mgr, NewReconciler(mgr)) +} + +// NewReconciler returns a new reconcile.Reconciler +func NewReconciler(mgr ctrl.Manager) reconcile.Reconciler { + recorder := mgr.GetEventRecorderFor(controllerName) + + InitExpectations(mgr.GetClient()) + + return &PodDeletionReconciler{ + Client: mgr.GetClient(), + recorder: recorder, + } +} + +func AddToMgr(mgr ctrl.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New(controllerName, mgr, controller.Options{ + MaxConcurrentReconciles: 5, + Reconciler: r, + }) + if err != nil { + return err + } + + err = c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForObject{}, &PredicateDeletionIndicatedPod{}) + if err != nil { + return err + } + + return nil +} + +// Reconcile aims to delete Pod through PodOpsLifecycle. It will watch Pod with label `kafed.kusionstack.io/to-delete`. +// If a Pod is labeled, controller will first trigger a deletion PodOpsLifecycle. If all conditions are satisfied, +// it will then delete Pod. +func (r *PodDeletionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + instance := &corev1.Pod{} + if err := r.Get(ctx, req.NamespacedName, instance); err != nil { + if !errors.IsNotFound(err) { + klog.Error("fail to find Pod %s: %s", req, err) + return reconcile.Result{}, err + } + + klog.Infof("Pod %s is deleted", req) + return ctrl.Result{}, activeExpectations.Delete(req.Namespace, req.Name) + } + + // if expectation not satisfied, shortcut this reconciling till informer cache is updated. + if satisfied, err := activeExpectations.IsSatisfied(instance); err != nil { + return ctrl.Result{}, err + } else if !satisfied { + klog.Warningf("Pod %s is not satisfied to reconcile.", req) + return ctrl.Result{}, nil + } + + if instance.DeletionTimestamp != nil { + return ctrl.Result{}, nil + } + + // if Pod is not begin a deletion PodOpsLifecycle, trigger it + if !podopslifecycle.IsDuringOps(OpsLifecycleAdapter, instance) { + if updated, err := podopslifecycle.Begin(r, OpsLifecycleAdapter, instance); err != nil { + return ctrl.Result{}, fmt.Errorf("fail to begin PodOpsLifecycle to delete Pod %s: %s", req, err) + } else if updated { + if err := activeExpectations.ExpectUpdate(instance, expectations.Pod, instance.Name, instance.ResourceVersion); err != nil { + return ctrl.Result{}, fmt.Errorf("fail to expect Pod updated after beginning PodOpsLifecycle to delete Pod %s: %s", req, err) + } + } + } + + // if Pod is allow to operate, delete it + if podopslifecycle.AllowOps(OpsLifecycleAdapter, instance) { + klog.Infof("try to delete Pod %s with deletion indication", req) + if err := r.Delete(context.TODO(), instance); err != nil { + return ctrl.Result{}, fmt.Errorf("fail to delete Pod %s with deletion indication: %s", req, err) + } else { + if err := activeExpectations.ExpectDelete(instance, expectations.Pod, instance.Name); err != nil { + return ctrl.Result{}, fmt.Errorf("fail to expect Pod %s deleted: %s", req, err) + } + } + } + + return ctrl.Result{}, nil +} diff --git a/pkg/controllers/poddeletion/poddeletion_controller_test.go b/pkg/controllers/poddeletion/poddeletion_controller_test.go new file mode 100644 index 00000000..d901c4f9 --- /dev/null +++ b/pkg/controllers/poddeletion/poddeletion_controller_test.go @@ -0,0 +1,228 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package poddeletion + +import ( + "context" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "kusionstack.io/kafed/apis" + appsv1alpha1 "kusionstack.io/kafed/apis/apps/v1alpha1" + "kusionstack.io/kafed/pkg/utils/inject" +) + +var ( + env *envtest.Environment + mgr manager.Manager + request chan reconcile.Request + + ctx context.Context + cancel context.CancelFunc + c client.Client +) + +var _ = Describe("Pod Deletion controller", func() { + + It("deletion reconcile", func() { + testcase := "delete-pod" + Expect(createNamespace(c, testcase)).Should(BeNil()) + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: "test", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "nginx:v1", + }, + }, + }, + } + + Expect(c.Create(context.TODO(), pod)).Should(BeNil()) + Eventually(func() error { + return c.Get(context.TODO(), types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, pod) + }, 5*time.Second, 1*time.Second).Should(BeNil()) + + Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { + pod.Labels = map[string]string{ + appsv1alpha1.PodDeletionIndicationLabelKey: "true", + } + return true + })).Should(BeNil()) + + time.After(3 * time.Second) + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, pod)).Should(BeNil()) + + // allow Pod to update + Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, OpsLifecycleAdapter.GetID()) + if pod.Labels == nil { + pod.Labels = map[string]string{} + } + pod.Labels[labelOperate] = "true" + return true + })).Should(BeNil()) + + // pod should be deleted + Eventually(func() error { + return c.Get(context.TODO(), types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, pod) + }, 5*time.Second, 1*time.Second).ShouldNot(BeNil()) + }) +}) + +func updatePodWithRetry(c client.Client, namespace, name string, updateFn func(pod *corev1.Pod) bool) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + pod := &corev1.Pod{} + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: name}, pod); err != nil { + return err + } + + if !updateFn(pod) { + return nil + } + + return c.Update(context.TODO(), pod) + }) +} + +func updatePodStatusWithRetry(c client.Client, namespace, name string, updateFn func(pod *corev1.Pod) bool) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + pod := &corev1.Pod{} + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: name}, pod); err != nil { + return err + } + + if !updateFn(pod) { + return nil + } + + return c.Status().Update(context.TODO(), pod) + }) +} + +func testReconcile(inner reconcile.Reconciler) (reconcile.Reconciler, chan reconcile.Request) { + requests := make(chan reconcile.Request, 5) + fn := reconcile.Func(func(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + result, err := inner.Reconcile(ctx, req) + if _, done := ctx.Deadline(); !done && len(requests) == 0 { + requests <- req + } + return result, err + }) + return fn, requests +} + +func TestPodDeletionController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "PodDeletionController Test Suite") +} + +var _ = BeforeSuite(func() { + By("bootstrapping test environment") + + ctx, cancel = context.WithCancel(context.TODO()) + logf.SetLogger(zap.New(zap.WriteTo(os.Stdout), zap.UseDevMode(true))) + + env = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, + } + env.ControlPlane.GetAPIServer().URL = &url.URL{ + Host: "127.0.0.1:10001", + } + config, err := env.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(config).NotTo(BeNil()) + + mgr, err = manager.New(config, manager.Options{ + MetricsBindAddress: "0", + NewCache: inject.NewCacheWithFieldIndex, + }) + Expect(err).NotTo(HaveOccurred()) + + scheme := mgr.GetScheme() + err = appsv1.SchemeBuilder.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + err = apis.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + c = mgr.GetClient() + + var r reconcile.Reconciler + r, request = testReconcile(NewReconciler(mgr)) + err = AddToMgr(mgr, r) + Expect(err).NotTo(HaveOccurred()) + + go func() { + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }() +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + + cancel() + + err := env.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +var _ = AfterEach(func() { + nsList := &corev1.NamespaceList{} + Expect(mgr.GetClient().List(context.Background(), nsList)).Should(BeNil()) + + for i := range nsList.Items { + if strings.HasPrefix(nsList.Items[i].Name, "test-") { + mgr.GetClient().Delete(context.TODO(), &nsList.Items[i]) + } + } +}) + +func createNamespace(c client.Client, namespaceName string) error { + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespaceName, + }, + } + + return c.Create(context.TODO(), ns) +} diff --git a/pkg/controllers/poddeletion/predict.go b/pkg/controllers/poddeletion/predict.go new file mode 100644 index 00000000..46883474 --- /dev/null +++ b/pkg/controllers/poddeletion/predict.go @@ -0,0 +1,59 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package poddeletion + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + + appsv1alpha1 "kusionstack.io/kafed/apis/apps/v1alpha1" +) + +type PredicateDeletionIndicatedPod struct { +} + +// Create returns true if the Create event should be processed +func (p *PredicateDeletionIndicatedPod) Create(e event.CreateEvent) bool { + return hasTerminatingLabel(e.Object) +} + +// Delete returns true if the Delete event should be processed +func (p *PredicateDeletionIndicatedPod) Delete(e event.DeleteEvent) bool { + return hasTerminatingLabel(e.Object) +} + +// Update returns true if the Update event should be processed +func (p *PredicateDeletionIndicatedPod) Update(e event.UpdateEvent) bool { + return hasTerminatingLabel(e.ObjectNew) +} + +// Generic returns true if the Generic event should be processed +func (p *PredicateDeletionIndicatedPod) Generic(e event.GenericEvent) bool { + return hasTerminatingLabel(e.Object) +} + +func hasTerminatingLabel(pod client.Object) bool { + if pod.GetLabels() == nil { + return false + } + + if _, exist := pod.GetLabels()[appsv1alpha1.PodDeletionIndicationLabelKey]; exist { + return true + } + + return false +} diff --git a/pkg/controllers/resourcecontext/expectation.go b/pkg/controllers/resourcecontext/expectation.go new file mode 100644 index 00000000..98440902 --- /dev/null +++ b/pkg/controllers/resourcecontext/expectation.go @@ -0,0 +1,55 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourcecontext + +import ( + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + + "kusionstack.io/kafed/pkg/controllers/utils/expectations" +) + +var ( + // activeExpectations is used to check the cache in informer is updated, before reconciling. + activeExpectations *expectations.ActiveExpectations +) + +func InitExpectations(c client.Client) { + activeExpectations = expectations.NewActiveExpectations(c) +} + +type ExpectationEventHandler struct { +} + +// Create is called in response to an create event - e.g. Pod Creation. +func (h *ExpectationEventHandler) Create(event.CreateEvent, workqueue.RateLimitingInterface) {} + +// Update is called in response to an update event - e.g. Pod Updated. +func (h *ExpectationEventHandler) Update(event.UpdateEvent, workqueue.RateLimitingInterface) {} + +// Delete is called in response to a delete event - e.g. Pod Deleted. +func (h *ExpectationEventHandler) Delete(e event.DeleteEvent, _ workqueue.RateLimitingInterface) { + if err := activeExpectations.Delete(e.Object.GetNamespace(), e.Object.GetName()); err != nil { + klog.Error("fail to delete expectation in ResourceContextController for %s/%s: %s", e.Object.GetNamespace(), e.Object.GetName(), err) + } +} + +// Generic is called in response to an event of an unknown type or a synthetic event triggered as a cron or +// external trigger request - e.g. reconcile Autoscaling, or a Webhook. +func (h *ExpectationEventHandler) Generic(event.GenericEvent, workqueue.RateLimitingInterface) {} diff --git a/pkg/controllers/resourcecontext/resourcecontext_controller.go b/pkg/controllers/resourcecontext/resourcecontext_controller.go new file mode 100644 index 00000000..066f4c7b --- /dev/null +++ b/pkg/controllers/resourcecontext/resourcecontext_controller.go @@ -0,0 +1,122 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourcecontext + +import ( + "context" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + appsv1alpha1 "kusionstack.io/kafed/apis/apps/v1alpha1" + "kusionstack.io/kafed/pkg/controllers/utils/expectations" +) + +const ( + controllerName = "resourcecontext-controller" +) + +// ResourceContextReconciler reconciles and reclaims a ResourceContext object +type ResourceContextReconciler struct { + client.Client + + recorder record.EventRecorder +} + +func Add(mgr ctrl.Manager) error { + return AddToMgr(mgr, NewReconciler(mgr)) +} + +// NewReconciler returns a new reconcile.Reconciler +func NewReconciler(mgr ctrl.Manager) reconcile.Reconciler { + recorder := mgr.GetEventRecorderFor(controllerName) + + InitExpectations(mgr.GetClient()) + + return &ResourceContextReconciler{ + Client: mgr.GetClient(), + recorder: recorder, + } +} + +func AddToMgr(mgr ctrl.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New(controllerName, mgr, controller.Options{ + MaxConcurrentReconciles: 5, + Reconciler: r, + }) + if err != nil { + return err + } + + err = c.Watch(&source.Kind{Type: &appsv1alpha1.ResourceContext{}}, &handler.EnqueueRequestForObject{}) + if err != nil { + return err + } + + // Watch for changes to maintain expectation + err = c.Watch(&source.Kind{Type: &appsv1alpha1.ResourceContext{}}, &ExpectationEventHandler{}) + if err != nil { + return err + } + + return nil +} + +// Reconcile aims to reclaim ResourceContext which is not in used which means the ResourceContext contains no Context. +func (r *ResourceContextReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + instance := &appsv1alpha1.ResourceContext{} + if err := r.Get(ctx, req.NamespacedName, instance); err != nil { + if !errors.IsNotFound(err) { + klog.Error("fail to find ResourceContext %s: %s", req, err) + return reconcile.Result{}, err + } + + klog.Infof("ResourceContext %s is deleted", req) + return ctrl.Result{}, activeExpectations.Delete(req.Namespace, req.Name) + } + + // if expectation not satisfied, shortcut this reconciling till informer cache is updated. + if satisfied, err := activeExpectations.IsSatisfied(instance); err != nil { + return ctrl.Result{}, err + } else if !satisfied { + klog.Warningf("ResourceContext %s is not satisfied to reconcile.", req) + return ctrl.Result{}, nil + } + + // if ResourceContext is empty, delete it + if len(instance.Spec.Contexts) == 0 { + klog.Infof("try to delete ResourceContext %s as empty", req) + if err := r.Delete(context.TODO(), instance); err != nil { + klog.Error("fail to delete ResourceContext %s: %s", req, err) + return ctrl.Result{}, err + } + if err := activeExpectations.ExpectDelete(instance, expectations.ResourceContext, instance.Name); err != nil { + klog.Error("fail to expect deletion after deleting ResourceContext %s: %s", req, err) + return ctrl.Result{}, err + } + } + + return ctrl.Result{}, nil +} diff --git a/pkg/controllers/resourcecontext/resourcecontext_controller_test.go b/pkg/controllers/resourcecontext/resourcecontext_controller_test.go new file mode 100644 index 00000000..5bff1470 --- /dev/null +++ b/pkg/controllers/resourcecontext/resourcecontext_controller_test.go @@ -0,0 +1,352 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourcecontext + +import ( + "context" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/util/sets" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "kusionstack.io/kafed/apis" + appsv1alpha1 "kusionstack.io/kafed/apis/apps/v1alpha1" + "kusionstack.io/kafed/pkg/controllers/collaset" + collasetutils "kusionstack.io/kafed/pkg/controllers/collaset/utils" + "kusionstack.io/kafed/pkg/controllers/poddeletion" + "kusionstack.io/kafed/pkg/utils/inject" +) + +var ( + env *envtest.Environment + mgr manager.Manager + request chan reconcile.Request + + ctx context.Context + cancel context.CancelFunc + c client.Client +) + +var _ = Describe("ResourceContext controller", func() { + + It("resource context reconcile", func() { + testcase := "test-reclaim" + Expect(createNamespace(c, testcase)).Should(BeNil()) + + cs := &appsv1alpha1.CollaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: "foo", + }, + Spec: appsv1alpha1.CollaSetSpec{ + Replicas: int32Pointer(2), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "foo", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "foo", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + + Expect(c.Create(context.TODO(), cs)).Should(BeNil()) + + podList := &corev1.PodList{} + Eventually(func() bool { + Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + return len(podList.Items) == 2 + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}, cs)).Should(BeNil()) + Expect(expectedStatusReplicas(c, cs, 0, 0, 0, 2, 2, 0, 0, 0)).Should(BeNil()) + + names := sets.NewString() + for _, pod := range podList.Items { + names.Insert(pod.Name) + // mark Pod to delete + Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { + pod.Labels[appsv1alpha1.PodDeletionIndicationLabelKey] = "true" + return true + })).Should(BeNil()) + // allow Pod to delete + Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, poddeletion.OpsLifecycleAdapter.GetID()) + pod.Labels[labelOperate] = "true" + return true + })).Should(BeNil()) + } + + // wait for all original Pods deleted + Eventually(func() bool { + Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + if len(podList.Items) != 2 { + return false + } + + for _, pod := range podList.Items { + if pod.DeletionTimestamp != nil { + // still in terminating + return false + } + } + + for _, pod := range podList.Items { + if names.Has(pod.Name) { + return false + } + } + + return true + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + + resourceContext := &appsv1alpha1.ResourceContext{} + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}, resourceContext)).Should(BeNil()) + + Expect(updateCollaSetWithRetry(c, cs.Namespace, cs.Name, func(cls *appsv1alpha1.CollaSet) bool { + cls.Spec.Replicas = int32Pointer(0) + return true + })).Should(BeNil()) + + for _, pod := range podList.Items { + // allow Pod to scale in + Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.ScaleInOpsLifecycleAdapter.GetID()) + pod.Labels[labelOperate] = "true" + return true + })).Should(BeNil()) + } + + Eventually(func() error { + return expectedStatusReplicas(c, cs, 0, 0, 0, 0, 0, 0, 0, 0) + }, 5*time.Second, 1*time.Second).Should(BeNil()) + + Eventually(func() error { + return c.Get(context.TODO(), types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}, resourceContext) + }, 500*time.Second, 1*time.Second).ShouldNot(BeNil()) + }) +}) + +func expectedStatusReplicas(c client.Client, cls *appsv1alpha1.CollaSet, scheduledReplicas, readyReplicas, availableReplicas, replicas, updatedReplicas, operatingReplicas, + updatedReadyReplicas, updatedAvailableReplicas int32) error { + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: cls.Namespace, Name: cls.Name}, cls); err != nil { + return err + } + + if cls.Status.ScheduledReplicas != scheduledReplicas { + return fmt.Errorf("scheduledReplicas got %d, expected %d", cls.Status.ScheduledReplicas, scheduledReplicas) + } + + if cls.Status.ReadyReplicas != readyReplicas { + return fmt.Errorf("readyReplicas got %d, expected %d", cls.Status.ReadyReplicas, readyReplicas) + } + + if cls.Status.AvailableReplicas != availableReplicas { + return fmt.Errorf("availableReplicas got %d, expected %d", cls.Status.AvailableReplicas, availableReplicas) + } + + if cls.Status.Replicas != replicas { + return fmt.Errorf("replicas got %d, expected %d", cls.Status.Replicas, replicas) + } + + if cls.Status.UpdatedReplicas != updatedReplicas { + return fmt.Errorf("updatedReplicas got %d, expected %d", cls.Status.UpdatedReplicas, updatedReplicas) + } + + if cls.Status.OperatingReplicas != operatingReplicas { + return fmt.Errorf("operatingReplicas got %d, expected %d", cls.Status.OperatingReplicas, operatingReplicas) + } + + if cls.Status.UpdatedReadyReplicas != updatedReadyReplicas { + return fmt.Errorf("updatedReadyReplicas got %d, expected %d", cls.Status.UpdatedReadyReplicas, updatedReadyReplicas) + } + + if cls.Status.UpdatedAvailableReplicas != updatedAvailableReplicas { + return fmt.Errorf("updatedAvailableReplicas got %d, expected %d", cls.Status.UpdatedAvailableReplicas, updatedAvailableReplicas) + } + + return nil +} + +func updateCollaSetWithRetry(c client.Client, namespace, name string, updateFn func(cls *appsv1alpha1.CollaSet) bool) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + cls := &appsv1alpha1.CollaSet{} + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: name}, cls); err != nil { + return err + } + + if !updateFn(cls) { + return nil + } + + return c.Update(context.TODO(), cls) + }) +} + +func updatePodWithRetry(c client.Client, namespace, name string, updateFn func(pod *corev1.Pod) bool) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + pod := &corev1.Pod{} + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: name}, pod); err != nil { + return err + } + + if !updateFn(pod) { + return nil + } + + return c.Update(context.TODO(), pod) + }) +} + +func testReconcile(inner reconcile.Reconciler) (reconcile.Reconciler, chan reconcile.Request) { + requests := make(chan reconcile.Request, 5) + fn := reconcile.Func(func(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + result, err := inner.Reconcile(ctx, req) + if _, done := ctx.Deadline(); !done && len(requests) == 0 { + requests <- req + } + return result, err + }) + return fn, requests +} + +func TestResourceContextController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "ResourceContext Test Suite") +} + +var _ = BeforeSuite(func() { + By("bootstrapping test environment") + + ctx, cancel = context.WithCancel(context.TODO()) + logf.SetLogger(zap.New(zap.WriteTo(os.Stdout), zap.UseDevMode(true))) + + env = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, + } + env.ControlPlane.GetAPIServer().URL = &url.URL{ + Host: "127.0.0.1:10001", + } + config, err := env.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(config).NotTo(BeNil()) + + mgr, err = manager.New(config, manager.Options{ + MetricsBindAddress: "0", + NewCache: inject.NewCacheWithFieldIndex, + }) + Expect(err).NotTo(HaveOccurred()) + + scheme := mgr.GetScheme() + err = appsv1.SchemeBuilder.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + err = apis.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + c = mgr.GetClient() + + var r reconcile.Reconciler + r, request = testReconcile(NewReconciler(mgr)) + err = AddToMgr(mgr, r) + Expect(err).NotTo(HaveOccurred()) + + r, request = testReconcile(collaset.NewReconciler(mgr)) + err = collaset.AddToMgr(mgr, r) + Expect(err).NotTo(HaveOccurred()) + + r, request = testReconcile(poddeletion.NewReconciler(mgr)) + err = poddeletion.AddToMgr(mgr, r) + Expect(err).NotTo(HaveOccurred()) + + go func() { + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }() +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + + cancel() + + err := env.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +var _ = AfterEach(func() { + csList := &appsv1alpha1.CollaSetList{} + Expect(mgr.GetClient().List(context.Background(), csList)).Should(BeNil()) + + for i := range csList.Items { + Expect(mgr.GetClient().Delete(context.TODO(), &csList.Items[i])).Should(BeNil()) + } + + nsList := &corev1.NamespaceList{} + Expect(mgr.GetClient().List(context.Background(), nsList)).Should(BeNil()) + + for i := range nsList.Items { + if strings.HasPrefix(nsList.Items[i].Name, "test-") { + mgr.GetClient().Delete(context.TODO(), &nsList.Items[i]) + } + } +}) + +func createNamespace(c client.Client, namespaceName string) error { + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespaceName, + }, + } + + return c.Create(context.TODO(), ns) +} + +func int32Pointer(val int32) *int32 { + return &val +} diff --git a/pkg/controllers/utils/expectations/active_expectation.go b/pkg/controllers/utils/expectations/active_expectation.go new file mode 100644 index 00000000..a971dc55 --- /dev/null +++ b/pkg/controllers/utils/expectations/active_expectation.go @@ -0,0 +1,428 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package expectations + +import ( + "context" + "fmt" + "strconv" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "kusionstack.io/kafed/apis/apps/v1alpha1" +) + +var ( + ResourceInitializers map[ExpectedReosurceType]func() client.Object +) + +type ExpectedReosurceType string + +const ( + Pod ExpectedReosurceType = "pod" + Pvc ExpectedReosurceType = "pvc" + CollaSet ExpectedReosurceType = "CollaSet" + ResourceContext ExpectedReosurceType = "ResourceContext" +) + +type ActiveExpectationAction int + +const ( + Create ActiveExpectationAction = 0 + Delete ActiveExpectationAction = 1 + Update ActiveExpectationAction = 3 +) + +func init() { + ResourceInitializers = map[ExpectedReosurceType]func() client.Object{ + Pod: func() client.Object { + return &corev1.Pod{} + }, + Pvc: func() client.Object { + return &corev1.PersistentVolumeClaim{} + }, + CollaSet: func() client.Object { + return &appsv1alpha1.CollaSet{} + }, + ResourceContext: func() client.Object { + return &appsv1alpha1.ResourceContext{} + }, + } +} + +func ActiveExpectationKeyFunc(object interface{}) (string, error) { + expectation, ok := object.(*ActiveExpectation) + if !ok { + return "", fmt.Errorf("fail to convert to active expectation") + } + return expectation.key, nil +} + +func NewActiveExpectations(client client.Client) *ActiveExpectations { + return &ActiveExpectations{ + Client: client, + subjects: cache.NewStore(ActiveExpectationKeyFunc), + } +} + +type ActiveExpectations struct { + client.Client + subjects cache.Store +} + +func (ae *ActiveExpectations) expect(subject metav1.Object, kind ExpectedReosurceType, name string, action ActiveExpectationAction) error { + if _, exist := ResourceInitializers[kind]; !exist { + panic(fmt.Sprintf("kind %s is not supported for Active Expectation", kind)) + } + + if action == Update { + panic("update action is not supported by this method") + } + + key := fmt.Sprintf("%s/%s", subject.GetNamespace(), subject.GetName()) + expectation, exist, err := ae.subjects.GetByKey(key) + if err != nil { + return fmt.Errorf("fail to get expectation for active expectations %s when expecting: %s", key, err) + } + + if !exist { + expectation = NewActiveExpectation(ae.Client, subject.GetNamespace(), key) + if err := ae.subjects.Add(expectation); err != nil { + return err + } + } + + if err := expectation.(*ActiveExpectation).expect(kind, name, action); err != nil { + return fmt.Errorf("fail to expect %s/%s for action %d: %s", kind, name, action, err) + } + + return nil +} + +func (ae *ActiveExpectations) ExpectCreate(subject metav1.Object, kind ExpectedReosurceType, name string) error { + return ae.expect(subject, kind, name, Create) +} + +func (ae *ActiveExpectations) ExpectDelete(subject metav1.Object, kind ExpectedReosurceType, name string) error { + return ae.expect(subject, kind, name, Delete) +} + +func (ae *ActiveExpectations) ExpectUpdate(subject metav1.Object, kind ExpectedReosurceType, name string, updatedResourceVersion string) error { + rv, err := strconv.ParseInt(updatedResourceVersion, 10, 64) + if err != nil { + panic(fmt.Sprintf("fail to parse resource version %s of resource %s/%s to int64 for subject %s/%s: %s", + updatedResourceVersion, kind, name, subject.GetNamespace(), subject.GetName(), err)) + } + + if _, exist := ResourceInitializers[kind]; !exist { + panic(fmt.Sprintf("kind %s is not supported for Active Expectation", kind)) + } + + key := fmt.Sprintf("%s/%s", subject.GetNamespace(), subject.GetName()) + expectation, exist, err := ae.subjects.GetByKey(key) + if err != nil { + return fmt.Errorf("fail to get expectation for active expectations %s when expecting: %s", key, err) + } + + if !exist { + expectation = NewActiveExpectation(ae.Client, subject.GetNamespace(), key) + if err := ae.subjects.Add(expectation); err != nil { + return err + } + } + + if err := expectation.(*ActiveExpectation).expectUpdate(kind, name, rv); err != nil { + return fmt.Errorf("fail to expect %s/%s for action %d, %s: %s", kind, name, Update, updatedResourceVersion, err) + } + + return nil +} + +func (ae *ActiveExpectations) IsSatisfied(subject metav1.Object) (satisfied bool, err error) { + key := fmt.Sprintf("%s/%s", subject.GetNamespace(), subject.GetName()) + expectation, exist, err := ae.subjects.GetByKey(key) + if err != nil { + return false, fmt.Errorf("fail to get expectation for active expectations %s when check satisfication: %s", key, err) + } + + if !exist { + return true, nil + } + + defer func() { + if satisfied { + ae.subjects.Delete(expectation) + } + }() + + satisfied, err = expectation.(*ActiveExpectation).isSatisfied() + if err != nil { + return false, err + } + + return +} + +func (ae *ActiveExpectations) DeleteItem(subject metav1.Object, kind ExpectedReosurceType, name string) error { + if _, exist := ResourceInitializers[kind]; !exist { + panic(fmt.Sprintf("kind %s is not supported for Active Expectation", kind)) + } + + key := fmt.Sprintf("%s/%s", subject.GetNamespace(), subject.GetName()) + expectation, exist, err := ae.subjects.GetByKey(key) + if err != nil { + return fmt.Errorf("fail to get expectation for active expectations %s when deleting name %s: %s", key, name, err) + } + + if !exist { + return nil + } + + item := expectation.(*ActiveExpectation) + if err := item.delete(string(kind), name); err != nil { + return fmt.Errorf("fail to delete %s/%s for key %s: %s", kind, name, key, err) + } + + if len(item.items.List()) == 0 { + ae.subjects.Delete(expectation) + } + + return nil +} + +func (ae *ActiveExpectations) DeleteByKey(key string) error { + expectation, exist, err := ae.subjects.GetByKey(key) + if err != nil { + return fmt.Errorf("fail to get expectation for active expectations %s when deleting: %s", key, err) + } + + if !exist { + return nil + } + + err = ae.subjects.Delete(expectation) + if err != nil { + return fmt.Errorf("fail to do delete expectation for active expectations %s when deleting: %s", key, err) + } + + return nil +} + +func (ae *ActiveExpectations) Delete(namespace, name string) error { + key := fmt.Sprintf("%s/%s", namespace, name) + return ae.DeleteByKey(key) +} + +func (ae *ActiveExpectations) GetExpectation(namespace, name string) (*ActiveExpectation, error) { + key := fmt.Sprintf("%s/%s", namespace, name) + expectation, exist, err := ae.subjects.GetByKey(key) + if err != nil { + return nil, fmt.Errorf("fail to get expectation for active expectations %s when getting: %s", key, err) + } + + if !exist { + return nil, nil + } + + return expectation.(*ActiveExpectation), nil +} + +func ActiveExpectationItemKeyFunc(object interface{}) (string, error) { + expectationItem, ok := object.(*ActiveExpectationItem) + if !ok { + return "", fmt.Errorf("fail to convert to active expectation item") + } + return expectationItem.Key, nil +} + +func NewActiveExpectation(client client.Client, namespace string, key string) *ActiveExpectation { + return &ActiveExpectation{ + Client: client, + namespace: namespace, + key: key, + items: cache.NewStore(ActiveExpectationItemKeyFunc), + recordTimestamp: time.Now(), + } +} + +type ActiveExpectation struct { + client.Client + namespace string + key string + items cache.Store + recordTimestamp time.Time +} + +func (ae *ActiveExpectation) expect(kind ExpectedReosurceType, name string, action ActiveExpectationAction) error { + if action == Update { + panic("update action is not supported by this method") + } + + key := fmt.Sprintf("%s/%s", kind, name) + item, exist, err := ae.items.GetByKey(key) + if err != nil { + return fmt.Errorf("fail to get active expectation item for %s when expecting: %s", key, err) + } + + ae.recordTimestamp = time.Now() + if !exist { + item = &ActiveExpectationItem{Client: ae.Client, Name: name, Kind: kind, Key: key, Action: action, RecordTimestamp: time.Now()} + if err := ae.items.Add(item); err != nil { + return err + } + + return nil + } + + item.(*ActiveExpectationItem).Action = action + item.(*ActiveExpectationItem).RecordTimestamp = time.Now() + return nil +} + +func (ae *ActiveExpectation) expectUpdate(kind ExpectedReosurceType, name string, resourceVersion int64) error { + key := fmt.Sprintf("%s/%s", kind, name) + item, exist, err := ae.items.GetByKey(key) + if err != nil { + return fmt.Errorf("fail to get active expectation item for %s when expecting: %s", key, err) + } + + ae.recordTimestamp = time.Now() + if !exist { + item = &ActiveExpectationItem{Client: ae.Client, Name: name, Kind: kind, Key: key, Action: Update, ResourceVersion: resourceVersion, RecordTimestamp: time.Now()} + if err := ae.items.Add(item); err != nil { + return err + } + + return nil + } + + ea := item.(*ActiveExpectationItem) + ea.Action = Update + ea.ResourceVersion = resourceVersion + ea.RecordTimestamp = time.Now() + + return nil +} + +func (ae *ActiveExpectation) isSatisfied() (satisfied bool, err error) { + items := ae.items.List() + + satisfied = true + for _, item := range items { + itemSatisfied, itemErr := func() (satisfied bool, err error) { + defer func() { + if satisfied { + ae.items.Delete(item) + } else if ae.recordTimestamp.Add(ExpectationsTimeout).Before(time.Now()) { + panic("expected panic for active expectation") + } + }() + + satisfied, err = item.(*ActiveExpectationItem).isSatisfied(ae.namespace) + if err != nil { + return false, err + } + + return satisfied, nil + }() + + if itemErr != nil && err == nil { + err = fmt.Errorf("fail to check satisfication for subject %s, item %s: %s", ae.key, item.(*ActiveExpectationItem).Key, err) + } + + satisfied = satisfied && itemSatisfied + } + + return satisfied, err +} + +func (ae *ActiveExpectation) delete(kind, name string) error { + key := fmt.Sprintf("%s/%s", kind, name) + item, exist, err := ae.items.GetByKey(key) + if err != nil { + return fmt.Errorf("fail to delete active expectation item for %s: %s", key, err) + } + + if !exist { + return nil + } + + if err := ae.items.Delete(item); err != nil { + return fmt.Errorf("fail to do delete active expectation item for %s: %s", key, err) + } + + return nil +} + +type ActiveExpectationItem struct { + client.Client + + Key string + Name string + Kind ExpectedReosurceType + Action ActiveExpectationAction + ResourceVersion int64 + RecordTimestamp time.Time +} + +func (i *ActiveExpectationItem) isSatisfied(namespace string) (bool, error) { + switch i.Action { + case Create: + resource := ResourceInitializers[i.Kind]() + if err := i.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: i.Name}, resource); err == nil { + return true, nil + } else if errors.IsNotFound(err) && i.RecordTimestamp.Add(30*time.Second).Before(time.Now()) { + // tolerate watch event missing, after 30s + return true, nil + } + case Delete: + resource := ResourceInitializers[i.Kind]() + if err := i.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: i.Name}, resource); err != nil { + if errors.IsNotFound(err) { + return true, nil + } + } else { + if resource.(metav1.Object).GetDeletionTimestamp() != nil { + return true, nil + } + } + case Update: + resource := ResourceInitializers[i.Kind]() + if err := i.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: i.Name}, resource); err == nil { + rv, err := strconv.ParseInt(resource.(metav1.Object).GetResourceVersion(), 10, 64) + if err != nil { + // true for error + return true, nil + } + if rv >= i.ResourceVersion { + return true, nil + } + } else { + if errors.IsNotFound(err) { + return true, nil + } + } + } + + return false, nil +} diff --git a/pkg/controllers/utils/expectations/active_expectation_test.go b/pkg/controllers/utils/expectations/active_expectation_test.go new file mode 100644 index 00000000..03aca21a --- /dev/null +++ b/pkg/controllers/utils/expectations/active_expectation_test.go @@ -0,0 +1,178 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package expectations + +import ( + "testing" + "time" + + "github.com/onsi/gomega" + "golang.org/x/net/context" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +var c client.Client + +var expectedRequest = reconcile.Request{NamespacedName: types.NamespacedName{Name: "foo", Namespace: "default"}} + +var depKey = types.NamespacedName{Name: "foo-deployment", Namespace: "default"} + +func Setup(t *testing.T) (g *gomega.GomegaWithT, r reconcile.Reconciler, mgr manager.Manager, client client.Client, stopFun func()) { + g = gomega.NewGomegaWithT(t) + mgr, err := manager.New(cfg, manager.Options{ + MetricsBindAddress: "0", + }) + g.Expect(err).NotTo(gomega.HaveOccurred()) + c = mgr.GetClient() + + stopMgr, mgrStopped := StartTestManager(mgr, g) + return g, r, mgr, mgr.GetClient(), func() { + close(stopMgr) + mgrStopped.Wait() + } +} + +func TestReconcileResponsePodEvent(t *testing.T) { + g, _, _, client, stopFunc := Setup(t) + defer func() { + stopFunc() + }() + + exp := NewActiveExpectations(client) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "parent", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "image:v1", + }, + }, + }, + } + g.Expect(client.Create(context.TODO(), pod)).Should(gomega.BeNil()) + + podA := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + GenerateName: "test-", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "image:v1", + }, + }, + }, + } + g.Expect(client.Create(context.TODO(), podA)).Should(gomega.BeNil()) + time.Sleep(3 * time.Second) + + podBName := "test-b" + g.Expect(exp.ExpectCreate(pod, Pod, podA.Name)).Should(gomega.BeNil()) + g.Expect(exp.ExpectCreate(pod, Pod, podBName)).Should(gomega.BeNil()) + expectation, err := exp.GetExpectation(pod.Namespace, pod.Name) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(len(expectation.items.List())).Should(gomega.BeEquivalentTo(2)) + + sa, err := exp.IsSatisfied(pod) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(sa).Should(gomega.BeFalse()) + expectation, err = exp.GetExpectation(pod.Namespace, pod.Name) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(len(expectation.items.List())).Should(gomega.BeEquivalentTo(1)) + + podB := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: podBName, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "image:v1", + }, + }, + }, + } + g.Expect(client.Create(context.TODO(), podB)).Should(gomega.BeNil()) + time.Sleep(3 * time.Second) + + sa, err = exp.IsSatisfied(pod) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(sa).Should(gomega.BeTrue()) + expectation, err = exp.GetExpectation(pod.Namespace, pod.Name) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(expectation).Should(gomega.BeNil()) + + // expected update + podB.Labels = map[string]string{ + "test": "test", + } + g.Expect(client.Update(context.TODO(), podB)).Should(gomega.BeNil()) + exp.ExpectUpdate(pod, Pod, pod.Name, pod.ResourceVersion) + time.Sleep(3 * time.Second) + + sa, err = exp.IsSatisfied(pod) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(sa).Should(gomega.BeTrue()) + expectation, err = exp.GetExpectation(pod.Namespace, pod.Name) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(expectation).Should(gomega.BeNil()) + + g.Expect(exp.ExpectDelete(pod, Pod, podBName)).Should(gomega.BeNil()) + g.Expect(exp.ExpectDelete(pod, Pod, podBName)).Should(gomega.BeNil()) + sa, err = exp.IsSatisfied(pod) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(sa).Should(gomega.BeFalse()) + + g.Expect(client.Delete(context.TODO(), podA)).Should(gomega.BeNil()) + time.Sleep(3 * time.Second) + + sa, err = exp.IsSatisfied(pod) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(sa).Should(gomega.BeFalse()) + expectation, err = exp.GetExpectation(pod.Namespace, pod.Name) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(len(expectation.items.List())).Should(gomega.BeEquivalentTo(1)) + + g.Expect(client.Delete(context.TODO(), podB)).Should(gomega.BeNil()) + time.Sleep(3 * time.Second) + + sa, err = exp.IsSatisfied(pod) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(sa).Should(gomega.BeTrue()) + expectation, err = exp.GetExpectation(pod.Namespace, pod.Name) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(expectation).Should(gomega.BeNil()) + + g.Expect(exp.ExpectCreate(pod, Pod, podA.Name)).Should(gomega.BeNil()) + g.Expect(exp.Delete(pod.Namespace, pod.Name)).Should(gomega.BeNil()) + expectation, err = exp.GetExpectation(pod.Namespace, pod.Name) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(expectation).Should(gomega.BeNil()) +} diff --git a/pkg/controllers/utils/expectations/expectations_suite_test.go b/pkg/controllers/utils/expectations/expectations_suite_test.go new file mode 100644 index 00000000..bbcef2c2 --- /dev/null +++ b/pkg/controllers/utils/expectations/expectations_suite_test.go @@ -0,0 +1,85 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package expectations + +import ( + "context" + stdlog "log" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "kusionstack.io/kafed/apis" +) + +var cfg *rest.Config + +func TestMain(m *testing.M) { + t := &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, + } + apis.AddToScheme(scheme.Scheme) + + var err error + if cfg, err = t.Start(); err != nil { + stdlog.Fatal(err) + } + + code := m.Run() + t.Stop() + os.Exit(code) +} + +// SetupTestReconcile returns a reconcile.Reconcile implementation that delegates to inner and +// writes the request to requests after Reconcile is finished. +func SetupTestReconcile(inner reconcile.Reconciler) (reconcile.Reconciler, chan reconcile.Request) { + requests := make(chan reconcile.Request, 100) + fn := reconcile.Func(func(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + result, err := inner.Reconcile(ctx, req) + if _, done := ctx.Deadline(); !done && len(requests) == 0 { + requests <- req + } + return result, err + }) + return fn, requests +} + +// StartTestManager adds recFn +func StartTestManager(mgr manager.Manager, g *gomega.GomegaWithT) (chan struct{}, *sync.WaitGroup) { + stop := make(chan struct{}) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + <-stop + cancel() + }() + + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + g.Expect(mgr.Start(ctx)).NotTo(gomega.HaveOccurred()) + }() + return stop, wg +} diff --git a/pkg/controllers/utils/expectations/resourceversion_expectation.go b/pkg/controllers/utils/expectations/resourceversion_expectation.go index 4999628a..2eb71414 100644 --- a/pkg/controllers/utils/expectations/resourceversion_expectation.go +++ b/pkg/controllers/utils/expectations/resourceversion_expectation.go @@ -92,8 +92,9 @@ func (r *ResourceVersionExpectation) ExpectUpdate(controllerKey string, resource } else if exists { exp.Set(resourceVersion) } else { - r.SetExpectations(controllerKey, resourceVersion) + return r.SetExpectations(controllerKey, resourceVersion) } + return nil } diff --git a/pkg/controllers/utils/pod_utils.go b/pkg/controllers/utils/pod_utils.go new file mode 100644 index 00000000..913847ba --- /dev/null +++ b/pkg/controllers/utils/pod_utils.go @@ -0,0 +1,267 @@ +/* +Copyright 2014 The Kubernetes Authors. +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "encoding/json" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/strategicpatch" + + appsv1alpha1 "kusionstack.io/kafed/apis/apps/v1alpha1" + revisionutils "kusionstack.io/kafed/pkg/controllers/utils/revision" +) + +func GetPodRevisionPatch(revision *appsv1.ControllerRevision) ([]byte, error) { + var raw map[string]interface{} + if err := json.Unmarshal([]byte(revision.Data.Raw), &raw); err != nil { + return nil, err + } + + spec := raw["spec"].(map[string]interface{}) + template := spec["template"].(map[string]interface{}) + patch, err := json.Marshal(template) + return patch, err +} + +func ApplyPatchFromRevision(pod *corev1.Pod, revision *appsv1.ControllerRevision) (*corev1.Pod, error) { + patch, err := GetPodRevisionPatch(revision) + if err != nil { + return nil, err + } + + clone := pod.DeepCopy() + patched, err := strategicpatch.StrategicMergePatch([]byte(runtime.EncodeOrDie(revisionutils.PodCodec, clone)), patch, clone) + if err != nil { + return nil, err + } + err = json.Unmarshal(patched, clone) + if err != nil { + return nil, err + } + return clone, nil +} + +// PatchToPod Use three way merge to get a updated pod. +func PatchToPod(currentRevisionPod, updateRevisionPod, currentPod *corev1.Pod) (*corev1.Pod, error) { + currentRevisionPodBytes, err := json.Marshal(currentRevisionPod) + if err != nil { + return nil, err + } + updateRevisionPodBytes, err := json.Marshal(updateRevisionPod) + + if err != nil { + return nil, err + } + + // 1. find the extra changes based on current revision + patch, err := strategicpatch.CreateTwoWayMergePatch(currentRevisionPodBytes, updateRevisionPodBytes, &corev1.Pod{}) + if err != nil { + return nil, err + } + + // 2. apply above changes to current pod + // We don't apply the diff between currentPod and currentRevisionPod to updateRevisionPod, + // because the PodTemplate changes should have the highest priority. + currentPodBytes, err := json.Marshal(currentPod) + if err != nil { + return nil, err + } + if updateRevisionPodBytes, err = strategicpatch.StrategicMergePatch(currentPodBytes, patch, &corev1.Pod{}); err != nil { + return nil, err + } + + newPod := &corev1.Pod{} + err = json.Unmarshal(updateRevisionPodBytes, newPod) + return newPod, err +} + +func NewPodFrom(owner metav1.Object, ownerRef *metav1.OwnerReference, revision *appsv1.ControllerRevision) (*corev1.Pod, error) { + pod, err := GetPodFromRevision(revision) + if err != nil { + return pod, err + } + + pod.Namespace = owner.GetNamespace() + pod.GenerateName = GetPodsPrefix(owner.GetName()) + pod.OwnerReferences = append(pod.OwnerReferences, *ownerRef) + + pod.Labels[appsv1.ControllerRevisionHashLabelKey] = revision.Name + + return pod, nil +} + +func GetPodFromRevision(revision *appsv1.ControllerRevision) (*corev1.Pod, error) { + pod, err := ApplyPatchFromRevision(&corev1.Pod{}, revision) + if err != nil { + return nil, err + } + + return pod, nil +} + +func GetPodsPrefix(controllerName string) string { + // use the dash (if the name isn't too long) to make the pod name a bit prettier + prefix := fmt.Sprintf("%s-", controllerName) + if len(apimachineryvalidation.NameIsDNSSubdomain(prefix, true)) != 0 { + prefix = controllerName + } + return prefix +} + +func ComparePod(l, r *corev1.Pod) bool { + // 1. Unassigned < assigned + // If only one of the pods is unassigned, the unassigned one is smaller + if l.Spec.NodeName != r.Spec.NodeName && (len(l.Spec.NodeName) == 0 || len(r.Spec.NodeName) == 0) { + return len(l.Spec.NodeName) == 0 + } + // 2. PodPending < PodUnknown < PodRunning + m := map[corev1.PodPhase]int{corev1.PodPending: 0, corev1.PodUnknown: 1, corev1.PodRunning: 2} + if m[l.Status.Phase] != m[r.Status.Phase] { + return m[l.Status.Phase] < m[r.Status.Phase] + } + // 3. Not ready < ready + // If only one of the pods is not ready, the not ready one is smaller + if IsPodReady(l) != IsPodReady(r) { + return !IsPodReady(l) + } + // TODO: take availability into account when we push minReadySeconds information from deployment into pods, + // see https://github.com/kubernetes/kubernetes/issues/22065 + // 4. Been ready for empty time < less time < more time + // If both pods are ready, the latest ready one is smaller + if IsPodReady(l) && IsPodReady(r) && !podReadyTime(l).Equal(podReadyTime(r)) { + return afterOrZero(podReadyTime(l), podReadyTime(r)) + } + // 5. Pods with containers with higher restart counts < lower restart counts + if maxContainerRestarts(l) != maxContainerRestarts(r) { + return maxContainerRestarts(l) > maxContainerRestarts(r) + } + // 6. Empty creation time pods < newer pods < older pods + if !l.CreationTimestamp.Equal(&r.CreationTimestamp) { + return afterOrZero(&l.CreationTimestamp, &r.CreationTimestamp) + } + return false +} + +func maxContainerRestarts(pod *corev1.Pod) int { + var maxRestarts int32 + for _, c := range pod.Status.ContainerStatuses { + if c.RestartCount > maxRestarts { + maxRestarts = c.RestartCount + } + } + return int(maxRestarts) +} + +func podReadyTime(pod *corev1.Pod) *metav1.Time { + if IsPodReady(pod) { + for _, c := range pod.Status.Conditions { + // we only care about pod ready conditions + if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue { + return &c.LastTransitionTime + } + } + } + return &metav1.Time{} +} + +// afterOrZero checks if time t1 is after time t2; if one of them +// is zero, the zero time is seen as after non-zero time. +func afterOrZero(t1, t2 *metav1.Time) bool { + if t1.Time.IsZero() || t2.Time.IsZero() { + return t1.Time.IsZero() + } + return t1.After(t2.Time) +} + +func IsPodScheduled(pod *corev1.Pod) bool { + return IsPodScheduledConditionTrue(pod.Status) +} + +func IsPodScheduledConditionTrue(status corev1.PodStatus) bool { + condition := GetPodScheduledCondition(status) + return condition != nil && condition.Status == corev1.ConditionTrue +} + +func GetPodScheduledCondition(status corev1.PodStatus) *corev1.PodCondition { + _, condition := GetPodCondition(&status, corev1.PodScheduled) + return condition +} + +// IsPodReady returns true if a pod is ready; false otherwise. +func IsPodReady(pod *corev1.Pod) bool { + return IsPodReadyConditionTrue(pod.Status) +} + +// IsPodReadyConditionTrue returns true if a pod is ready; false otherwise. +func IsPodReadyConditionTrue(status corev1.PodStatus) bool { + condition := GetPodReadyCondition(status) + return condition != nil && condition.Status == corev1.ConditionTrue +} + +// GetPodReadyCondition extracts the pod ready condition from the given status and returns that. +// Returns nil if the condition is not present. +func GetPodReadyCondition(status corev1.PodStatus) *corev1.PodCondition { + _, condition := GetPodCondition(&status, corev1.PodReady) + return condition +} + +// GetPodCondition extracts the provided condition from the given status and returns that. +// Returns nil and -1 if the condition is not present, and the index of the located condition. +func GetPodCondition(status *corev1.PodStatus, conditionType corev1.PodConditionType) (int, *corev1.PodCondition) { + if status == nil { + return -1, nil + } + return GetPodConditionFromList(status.Conditions, conditionType) +} + +// GetPodConditionFromList extracts the provided condition from the given list of condition and +// returns the index of the condition and the condition. Returns -1 and nil if the condition is not present. +func GetPodConditionFromList(conditions []corev1.PodCondition, conditionType corev1.PodConditionType) (int, *corev1.PodCondition) { + if conditions == nil { + return -1, nil + } + for i := range conditions { + if conditions[i].Type == conditionType { + return i, &conditions[i] + } + } + return -1, nil +} + +func IsServiceAvailable(pod *corev1.Pod) bool { + if pod.Labels == nil { + return false + } + + _, exist := pod.Labels[appsv1alpha1.PodServiceAvailableLabel] + return exist +} + +func IsPodUpdatedRevision(pod *corev1.Pod, revision string) bool { + if pod.Labels == nil { + return false + } + + return pod.Labels[appsv1.ControllerRevisionHashLabelKey] == revision +} diff --git a/pkg/controllers/utils/podopslifecycle/adapter.go b/pkg/controllers/utils/podopslifecycle/adapter.go new file mode 100644 index 00000000..5e94ab71 --- /dev/null +++ b/pkg/controllers/utils/podopslifecycle/adapter.go @@ -0,0 +1,47 @@ +/* + Copyright 2023 The KusionStack Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package podopslifecycle + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type OperationType string + +var ( + OpsLifecycleTypeUpdate OperationType = "update" + OpsLifecycleTypeScaleIn OperationType = "scaleIn" + OpsLifecycleTypeDelete OperationType = "delete" +) + +// LifecycleAdapter helps CRD Operators to easily access PodOpsLifecycle +type LifecycleAdapter interface { + // GetID indicates ID of one PodOpsLifecycle + GetID() string + + // GetType indicates type for an Operator + GetType() OperationType + + // AllowMultiType indicates whether multiple IDs which have the same Type are allowed + AllowMultiType() bool + + // WhenBegin will be executed when begin a lifecycle + WhenBegin(pod client.Object) (bool, error) + + // WhenFinish will be executed when finish a lifecycle + WhenFinish(pod client.Object) (bool, error) +} diff --git a/pkg/controllers/utils/podopslifecycle/utils.go b/pkg/controllers/utils/podopslifecycle/utils.go new file mode 100644 index 00000000..f382eea4 --- /dev/null +++ b/pkg/controllers/utils/podopslifecycle/utils.go @@ -0,0 +1,191 @@ +/* + Copyright 2023 The KusionStack Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package podopslifecycle + +import ( + "context" + "fmt" + "strings" + "time" + + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" + + "kusionstack.io/kafed/apis/apps/v1alpha1" +) + +// IsDuringOps decides whether the Pod is during ops or not +// DuringOps means the Pod's OpsLifecycle phase is in or after PreCheck phase and before Finish phase. +func IsDuringOps(adapter LifecycleAdapter, obj client.Object) bool { + _, hasID := checkOperatingID(adapter, obj) + _, hasType := checkOperationType(adapter, obj) + + return hasID && hasType +} + +// Begin is used for an CRD Operator to begin a lifecycle +func Begin(c client.Client, adapter LifecycleAdapter, obj client.Object) (updated bool, err error) { + if obj.GetLabels() == nil { + obj.SetLabels(map[string]string{}) + } + + operatingID, hasID := checkOperatingID(adapter, obj) + operationType, hasType := checkOperationType(adapter, obj) + var needUpdate bool + + // ensure operatingID and operationType + if hasID && hasType { + if operationType != adapter.GetType() { + err = fmt.Errorf("operatingID %s already has operationType %s", operatingID, operationType) + return + } + } else { + // check another id/type = this.type + currentTypeIDs := queryByOperationType(adapter, obj) + if currentTypeIDs != nil && currentTypeIDs.Len() > 0 && !adapter.AllowMultiType() { + err = fmt.Errorf("operationType %s exists: %v", adapter.GetType(), currentTypeIDs) + return + } + + if !hasID { + needUpdate = true + setOperatingID(adapter, obj) + } + if !hasType { + needUpdate = true + setOperationType(adapter, obj) + } + } + + updated, err = adapter.WhenBegin(obj) + if err != nil { + return + } + + if needUpdate || updated { + err = c.Update(context.Background(), obj) + return true, err + } + + return false, nil +} + +// AllowOps is used to check whether the PodOpsLifecycle phase is in UPGRADE to do following operations. +func AllowOps(adapter LifecycleAdapter, obj client.Object) (allow bool) { + if !IsDuringOps(adapter, obj) { + return false + } + + _, started := checkOperate(adapter, obj) + return started +} + +// Finish is used for an CRD Operator to finish a lifecycle +func Finish(c client.Client, adapter LifecycleAdapter, obj client.Object) (updated bool, err error) { + operatingID, hasID := checkOperatingID(adapter, obj) + operationType, hasType := checkOperationType(adapter, obj) + + if hasType && operationType != adapter.GetType() { + return false, fmt.Errorf("operatingID %s has invalid operationType %s", operatingID, operationType) + } + + var needUpdate bool + if hasID || hasType { + needUpdate = true + deleteOperatingID(adapter, obj) + deleteOperationType(adapter, obj) + } + + updated, err = adapter.WhenFinish(obj) + if err != nil { + return + } + if needUpdate || updated { + err = c.Update(context.Background(), obj) + if err != nil { + return + } + } + + return needUpdate, err +} + +func checkOperatingID(adapter LifecycleAdapter, obj client.Object) (val string, ok bool) { + labelID := fmt.Sprintf("%s/%s", v1alpha1.PodOperatingLabelPrefix, adapter.GetID()) + _, ok = obj.GetLabels()[labelID] + return adapter.GetID(), ok +} + +func checkOperationType(adapter LifecycleAdapter, obj client.Object) (val OperationType, ok bool) { + labelType := fmt.Sprintf("%s/%s", v1alpha1.PodOperationTypeLabelPrefix, adapter.GetID()) + + var labelVal string + labelVal, ok = obj.GetLabels()[labelType] + val = OperationType(labelVal) + + return +} + +func checkOperate(adapter LifecycleAdapter, obj client.Object) (val string, ok bool) { + labelOperate := fmt.Sprintf("%s/%s", v1alpha1.PodOperateLabelPrefix, adapter.GetID()) + val, ok = obj.GetLabels()[labelOperate] + return +} + +func setOperatingID(adapter LifecycleAdapter, obj client.Object) (val string, ok bool) { + labelID := fmt.Sprintf("%s/%s", v1alpha1.PodOperatingLabelPrefix, adapter.GetID()) + obj.GetLabels()[labelID] = fmt.Sprintf("%d", time.Now().UnixNano()) + return +} + +func setOperationType(adapter LifecycleAdapter, obj client.Object) (val string, ok bool) { + labelType := fmt.Sprintf("%s/%s", v1alpha1.PodOperationTypeLabelPrefix, adapter.GetID()) + obj.GetLabels()[labelType] = string(adapter.GetType()) + return +} + +// setOperate only for test +func setOperate(adapter LifecycleAdapter, obj client.Object) (val string, ok bool) { + labelOperate := fmt.Sprintf("%s/%s", v1alpha1.PodOperateLabelPrefix, adapter.GetID()) + obj.GetLabels()[labelOperate] = "true" + return +} + +func deleteOperatingID(adapter LifecycleAdapter, obj client.Object) (val string, ok bool) { + labelID := fmt.Sprintf("%s/%s", v1alpha1.PodOperatingLabelPrefix, adapter.GetID()) + delete(obj.GetLabels(), labelID) + return +} + +func deleteOperationType(adapter LifecycleAdapter, obj client.Object) (val string, ok bool) { + labelType := fmt.Sprintf("%s/%s", v1alpha1.PodOperationTypeLabelPrefix, adapter.GetID()) + delete(obj.GetLabels(), labelType) + return +} + +func queryByOperationType(adapter LifecycleAdapter, obj client.Object) sets.String { + res := sets.String{} + valType := adapter.GetType() + + for k, v := range obj.GetLabels() { + if strings.HasPrefix(k, v1alpha1.PodOperationTypeLabelPrefix) && v == string(valType) { + res.Insert(k) + } + } + + return res +} diff --git a/pkg/controllers/utils/podopslifecycle/utils_test.go b/pkg/controllers/utils/podopslifecycle/utils_test.go new file mode 100644 index 00000000..3fb988f6 --- /dev/null +++ b/pkg/controllers/utils/podopslifecycle/utils_test.go @@ -0,0 +1,156 @@ +/* + Copyright 2023 The KusionStack Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package podopslifecycle + +import ( + "context" + "fmt" + "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/runtime" + "reflect" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + + "kusionstack.io/kafed/apis/apps/v1alpha1" +) + +const ( + mockLabelKey = "mockLabel" + mockLabelValue = "mockLabelValue" + testNamespace = "default" + testName = "pod-1" +) + +var ( + allowTypes = false + scheme = runtime.NewScheme() +) + +func init() { + corev1.AddToScheme(scheme) +} + +func TestLifecycle(t *testing.T) { + c := fake.NewClientBuilder().WithScheme(scheme).Build() + g := gomega.NewGomegaWithT(t) + + a := &mockAdapter{id: "id-1", operationType: "type-1"} + b := &mockAdapter{id: "id-2", operationType: "type-1"} + + inputs := []struct { + hasOperating, hasConflictID bool + started bool + err error + allow bool + }{ + { + hasOperating: false, + started: false, + }, + { + hasOperating: true, + started: false, + }, + { + hasConflictID: true, + started: false, + err: fmt.Errorf("operationType %s exists: %v", a.GetType(), sets.NewString(fmt.Sprintf("%s/%s", v1alpha1.PodOperationTypeLabelPrefix, b.GetID()))), + }, + { + hasConflictID: true, + started: false, + allow: true, + err: nil, + }, + } + + for i, input := range inputs { + allowTypes = input.allow + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: fmt.Sprintf("%s-%d", testName, i), + Labels: map[string]string{}, + Annotations: map[string]string{}, + }, + } + g.Expect(c.Create(context.TODO(), pod)).Should(gomega.BeNil()) + + if input.hasOperating { + setOperatingID(a, pod) + setOperationType(a, pod) + a.WhenBegin(pod) + } + + if input.hasConflictID { + setOperatingID(b, pod) + setOperationType(b, pod) + } + + started, err := Begin(c, a, pod) + g.Expect(reflect.DeepEqual(err, input.err)).Should(gomega.BeTrue()) + if err != nil { + continue + } + g.Expect(pod.Labels[mockLabelKey]).Should(gomega.BeEquivalentTo(mockLabelValue)) + + setOperate(a, pod) + started, err = Begin(c, a, pod) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(started).Should(gomega.BeTrue()) + g.Expect(IsDuringOps(a, pod)).Should(gomega.BeTrue()) + + finished, err := Finish(c, a, pod) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(finished).Should(gomega.BeTrue()) + g.Expect(pod.Labels[mockLabelKey]).Should(gomega.BeEquivalentTo("")) + g.Expect(IsDuringOps(a, pod)).Should(gomega.BeFalse()) + } +} + +type mockAdapter struct { + id string + operationType OperationType +} + +func (m *mockAdapter) GetID() string { + return m.id +} + +func (m *mockAdapter) GetType() OperationType { + return m.operationType +} + +func (m *mockAdapter) AllowMultiType() bool { + return allowTypes +} + +func (m *mockAdapter) WhenBegin(pod client.Object) (bool, error) { + pod.GetLabels()[mockLabelKey] = mockLabelValue + return true, nil +} + +func (m *mockAdapter) WhenFinish(pod client.Object) (bool, error) { + delete(pod.GetLabels(), mockLabelKey) + return true, nil +} diff --git a/pkg/controllers/utils/refmanager/ref_manager.go b/pkg/controllers/utils/refmanager/ref_manager.go new file mode 100644 index 00000000..eca5d5a4 --- /dev/null +++ b/pkg/controllers/utils/refmanager/ref_manager.go @@ -0,0 +1,201 @@ +/* +Copyright 2016 The Kubernetes Authors. +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "fmt" + "sync" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +type RefManager struct { + client client.Writer + selector labels.Selector + owner metav1.Object + schema *runtime.Scheme + + once sync.Once + canAdoptErr error +} + +func NewRefManager(client client.Writer, selector *metav1.LabelSelector, owner metav1.Object, schema *runtime.Scheme) (*RefManager, error) { + s, err := metav1.LabelSelectorAsSelector(selector) + if err != nil { + return nil, err + } + + return &RefManager{ + client: client, + selector: s, + owner: owner, + schema: schema, + }, nil +} + +func (mgr *RefManager) ClaimOwned(objs []client.Object) ([]client.Object, error) { + match := func(obj metav1.Object) bool { + return mgr.selector.Matches(labels.Set(obj.GetLabels())) + } + + var claimObjs []client.Object + var errList []error + for _, obj := range objs { + ok, err := mgr.ClaimObject(obj, match) + if err != nil { + errList = append(errList, err) + continue + } + if ok { + claimObjs = append(claimObjs, obj) + } + } + return claimObjs, utilerrors.NewAggregate(errList) +} + +func (mgr *RefManager) canAdoptOnce() error { + mgr.once.Do(func() { + mgr.canAdoptErr = mgr.canAdopt() + }) + + return mgr.canAdoptErr +} + +func (mgr *RefManager) canAdopt() error { + if mgr.owner.GetDeletionTimestamp() != nil { + return fmt.Errorf("%v/%v has just been deleted at %v", + mgr.owner.GetNamespace(), mgr.owner.GetName(), mgr.owner.GetDeletionTimestamp()) + } + + return nil +} + +func (mgr *RefManager) adopt(obj client.Object) error { + if err := mgr.canAdoptOnce(); err != nil { + return fmt.Errorf("can't adopt Object %v/%v (%v): %v", obj.GetNamespace(), obj.GetName(), obj.GetUID(), err) + } + + if mgr.schema == nil { + return nil + } + + if err := controllerutil.SetControllerReference(mgr.owner, obj, mgr.schema); err != nil { + return fmt.Errorf("can't set Object %v/%v (%v) owner reference: %v", obj.GetNamespace(), obj.GetName(), obj.GetUID(), err) + } + + if err := mgr.client.Update(context.TODO(), obj); err != nil { + return fmt.Errorf("can't update Object %v/%v (%v) owner reference: %v", obj.GetNamespace(), obj.GetName(), obj.GetUID(), err) + } + return nil +} + +func (mgr *RefManager) release(obj client.Object) error { + idx := -1 + for i, ref := range obj.GetOwnerReferences() { + if ref.UID == mgr.owner.GetUID() { + idx = i + break + } + } + if idx > -1 { + obj.SetOwnerReferences(append(obj.GetOwnerReferences()[:idx], obj.GetOwnerReferences()[idx+1:]...)) + if err := mgr.client.Update(context.TODO(), obj); err != nil { + return fmt.Errorf("can't remove Pod %v/%v (%v) owner reference %v/%v (%v): %v", + obj.GetNamespace(), obj.GetName(), obj.GetUID(), obj.GetNamespace(), obj.GetName(), mgr.owner.GetUID(), err) + } + } + + return nil +} + +// ClaimObject tries to take ownership of an object for this controller. +// +// It will reconcile the following: +// - Adopt orphans if the match function returns true. +// - Release owned objects if the match function returns false. +// +// A non-nil error is returned if some form of reconciliation was attempted and +// failed. Usually, controllers should try again later in case reconciliation +// is still needed. +// +// If the error is nil, either the reconciliation succeeded, or no +// reconciliation was necessary. The returned boolean indicates whether you now +// own the object. +// +// No reconciliation will be attempted if the controller is being deleted. +func (mgr *RefManager) ClaimObject(obj client.Object, match func(metav1.Object) bool) (bool, error) { + controllerRef := metav1.GetControllerOf(obj) + if controllerRef != nil { + if controllerRef.UID != mgr.owner.GetUID() { + // Owned by someone else. Ignore. + return false, nil + } + if match(obj) { + // We already own it and the selector matches. + // Return true (successfully claimed) before checking deletion timestamp. + // We're still allowed to claim things we already own while being deleted + // because doing so requires taking no actions. + return true, nil + } + // Owned by us but selector doesn't match. + // Try to release, unless we're being deleted. + if mgr.owner.GetDeletionTimestamp() != nil { + return false, nil + } + if err := mgr.release(obj); err != nil { + // If the pod no longer exists, ignore the error. + if apierrors.IsNotFound(err) { + return false, nil + } + // Either someone else released it, or there was a transient error. + // The controller should requeue and try again if it's still stale. + return false, err + } + // Successfully released. + return false, nil + } + + // It's an orphan. + if mgr.owner.GetDeletionTimestamp() != nil || !match(obj) { + // Ignore if we're being deleted or selector doesn't match. + return false, nil + } + if obj.GetDeletionTimestamp() != nil { + // Ignore if the object is being deleted + return false, nil + } + // Selector matches. Try to adopt. + if err := mgr.adopt(obj); err != nil { + // If the pod no longer exists, ignore the error. + if apierrors.IsNotFound(err) { + return false, nil + } + // Either someone else claimed it first, or there was a transient error. + // The controller should requeue and try again if it's still orphaned. + return false, err + } + // Successfully adopted. + return true, nil +} diff --git a/pkg/controllers/utils/refmanager/ref_manager_test.go b/pkg/controllers/utils/refmanager/ref_manager_test.go new file mode 100644 index 00000000..f95b32c4 --- /dev/null +++ b/pkg/controllers/utils/refmanager/ref_manager_test.go @@ -0,0 +1,148 @@ +/* +Copyright 2016 The Kubernetes Authors. +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "testing" + + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "kusionstack.io/kafed/apis/apps/v1alpha1" +) + +func Test(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + scheme := runtime.NewScheme() + appsv1alpha1.AddToScheme(scheme) + + testcase := "ref-manager" + cs := &appsv1alpha1.CollaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "default", + UID: types.UID("fake-uid"), + }, + Spec: appsv1alpha1.CollaSetSpec{ + Replicas: int32Pointer(2), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + "case": testcase, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "foo", + "case": testcase, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + + pods := []corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "default", + Labels: map[string]string{ + "app": "foo", + "case": testcase, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx:v1", + }, + }, + }, + }, + } + + ref, err := NewRefManager(&MockClient{}, cs.Spec.Selector, cs, scheme) + g.Expect(err).Should(gomega.BeNil()) + + podObjects := make([]client.Object, len(pods)) + for i := range pods { + podObjects[i] = &pods[i] + } + + // adopt orphan pod + podObjects, err = ref.ClaimOwned(podObjects) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(len(podObjects)).Should(gomega.BeEquivalentTo(1)) + + ownerRef := metav1.GetControllerOf(podObjects[0]) + g.Expect(ownerRef).ShouldNot(gomega.BeNil()) + g.Expect(ownerRef.UID).Should(gomega.BeEquivalentTo(cs.UID)) + + // release pod not selected + labels := podObjects[0].GetLabels() + delete(labels, "app") + podObjects[0].SetLabels(labels) + claimedPodObjects, err := ref.ClaimOwned(podObjects) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(len(claimedPodObjects)).Should(gomega.BeEquivalentTo(0)) + ownerRef = metav1.GetControllerOf(podObjects[0]) + g.Expect(ownerRef).Should(gomega.BeNil()) +} + +type MockClient struct { +} + +func (c *MockClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + return nil +} + +func (c *MockClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + return nil +} + +func (c *MockClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + return nil +} + +func (c *MockClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + return nil +} + +func (c *MockClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + return nil +} + +func int32Pointer(val int32) *int32 { + return &val +} diff --git a/pkg/controllers/utils/revision/hash.go b/pkg/controllers/utils/revision/hash.go new file mode 100644 index 00000000..6a4dda18 --- /dev/null +++ b/pkg/controllers/utils/revision/hash.go @@ -0,0 +1,45 @@ +/* +Copyright 2015 The Kubernetes Authors. +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package revision + +import ( + "hash" + "k8s.io/klog/v2" + + "github.com/davecgh/go-spew/spew" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes/scheme" +) + +var PodCodec = scheme.Codecs.LegacyCodec(corev1.SchemeGroupVersion) + +// DeepHashObject writes specified object to hash using the spew library +// which follows pointers and prints actual values of the nested objects +// ensuring the hash does not change when a pointer changes. +func DeepHashObject(hasher hash.Hash, objectToWrite interface{}) { + hasher.Reset() + printer := spew.ConfigState{ + Indent: " ", + SortKeys: true, + DisableMethods: true, + SpewKeys: true, + } + if _, err := printer.Fprintf(hasher, "%#v", objectToWrite); err != nil { + klog.Error("fail to deep hash: %s", err) + } +} diff --git a/pkg/controllers/utils/revision/revision_manager.go b/pkg/controllers/utils/revision/revision_manager.go new file mode 100644 index 00000000..89cf9744 --- /dev/null +++ b/pkg/controllers/utils/revision/revision_manager.go @@ -0,0 +1,437 @@ +/* +Copyright 2017 The Kubernetes Authors. +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package revision + +import ( + "bytes" + "context" + "fmt" + "hash/fnv" + "sort" + "strconv" + + apps "k8s.io/api/apps/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + + refmanagerutil "kusionstack.io/kafed/pkg/controllers/utils/refmanager" +) + +const ControllerRevisionHashLabel = "controller.kubernetes.io/hash" + +type OwnerAdapter interface { + GetSelector(obj metav1.Object) *metav1.LabelSelector + GetCollisionCount(obj metav1.Object) *int32 + GetHistoryLimit(obj metav1.Object) int32 + GetPatch(obj metav1.Object) ([]byte, error) + GetSelectorLabels(obj metav1.Object) map[string]string + GetCurrentRevision(obj metav1.Object) string + IsInUsed(obj metav1.Object, controllerRevision string) bool +} + +func NewRevisionManager(client client.Client, scheme *runtime.Scheme, ownerGetter OwnerAdapter) *RevisionManager { + return &RevisionManager{ + Client: client, + scheme: scheme, + ownerGetter: ownerGetter, + } +} + +type RevisionManager struct { + client.Client + + scheme *runtime.Scheme + ownerGetter OwnerAdapter +} + +// controlledHistories returns all ControllerRevisions controlled by the given DaemonSet. +// This also reconciles ControllerRef by adopting/orphaning. +// Note that returned histories are pointers to objects in the cache. +// If you want to modify one, you need to deep-copy it first. +func controlledHistories(c client.Client, owner client.Object, labelSelector *metav1.LabelSelector, scheme *runtime.Scheme) ([]*apps.ControllerRevision, error) { + // List all histories to include those that don't match the selector anymore + // but have a ControllerRef pointing to the controller. + selector, err := metav1.LabelSelectorAsSelector(labelSelector) + if err != nil { + return nil, err + } + histories := &apps.ControllerRevisionList{} + if err := c.List(context.TODO(), histories, &client.ListOptions{Namespace: owner.GetNamespace(), LabelSelector: selector}); err != nil { + return nil, err + } + + // Use ControllerRefManager to adopt/orphan as needed. + cm, err := refmanagerutil.NewRefManager(c, labelSelector, owner, scheme) + if err != nil { + return nil, err + } + mts := make([]client.Object, len(histories.Items)) + for i, pod := range histories.Items { + mts[i] = pod.DeepCopy() + } + claims, err := cm.ClaimOwned(mts) + if err != nil { + return nil, err + } + + claimHistories := make([]*apps.ControllerRevision, len(claims)) + for i, mt := range claims { + claimHistories[i] = mt.(*apps.ControllerRevision) + } + + return claimHistories, nil +} + +// ConstructRevisions returns the current and update ControllerRevisions for set. It also +// returns a collision count that records the number of name collisions set saw when creating +// new ControllerRevisions. This count is incremented on every name collision and is used in +// building the ControllerRevision names for name collision avoidance. This method may create +// a new revision, or modify the Revision of an existing revision if an update to set is detected. +// This method expects that revisions is sorted when supplied. +func (rm *RevisionManager) ConstructRevisions(set client.Object, dryRun bool) (*apps.ControllerRevision, *apps.ControllerRevision, []*apps.ControllerRevision, *int32, bool, error) { + var currentRevision, updateRevision *apps.ControllerRevision + revisions, err := controlledHistories(rm.Client, set, rm.ownerGetter.GetSelector(set), rm.scheme) + if err != nil { + return currentRevision, updateRevision, nil, rm.ownerGetter.GetCollisionCount(set), false, err + } + + SortControllerRevisions(revisions) + if cleanedRevision, err := rm.cleanExpiredRevision(set, &revisions); err != nil { + return currentRevision, updateRevision, nil, rm.ownerGetter.GetCollisionCount(set), false, err + } else { + revisions = *cleanedRevision + } + + collisionCount := new(int32) + if rm.ownerGetter.GetCollisionCount(set) != nil { + collisionCount = rm.ownerGetter.GetCollisionCount(set) + } + // create a new revision from the current set + updateRevision, err = rm.newRevision(set, nextRevision(revisions), collisionCount) + if err != nil { + return nil, nil, nil, collisionCount, false, err + } + + // find any equivalent revisions + equalRevisions := FindEqualRevisions(revisions, updateRevision) + equalCount := len(equalRevisions) + revisionCount := len(revisions) + + createNewRevision := false + if equalCount > 0 && EqualRevision(revisions[revisionCount-1], equalRevisions[equalCount-1]) { + // if the equivalent revision is immediately prior the update revision has not changed + updateRevision = revisions[revisionCount-1] + } else if equalCount > 0 { + // if the equivalent revision is not immediately prior we will roll back by incrementing the + // Revision of the equivalent revision + equalRevisions[equalCount-1].Revision = updateRevision.Revision + rvc := equalRevisions[equalCount-1] + err = rm.Update(context.TODO(), rvc) + if err != nil { + return nil, nil, nil, collisionCount, false, err + } + equalRevisions[equalCount-1] = rvc + updateRevision = equalRevisions[equalCount-1] + } else { + if !dryRun { + //if there is no equivalent revision we create a new one + updateRevision, err = rm.createControllerRevision(context.TODO(), set, updateRevision, collisionCount) + if err != nil { + return nil, nil, nil, collisionCount, false, err + } + } + + createNewRevision = true + } + + // attempt to find the revision that corresponds to the current revision + for i := range revisions { + if revisions[i].Name == rm.ownerGetter.GetCurrentRevision(set) { + currentRevision = revisions[i] + } + } + + // if the current revision is nil we initialize the history by setting it to the update revision + if currentRevision == nil { + currentRevision = updateRevision + } + + return currentRevision, updateRevision, revisions, collisionCount, createNewRevision, nil +} + +func (rm *RevisionManager) cleanExpiredRevision(cd metav1.Object, sortedRevisions *[]*apps.ControllerRevision) (*[]*apps.ControllerRevision, error) { + limit := int(rm.ownerGetter.GetHistoryLimit(cd)) + if limit <= 0 { + limit = 10 + } + + // reserve 2 extra unused revisions for diagnose + exceedNum := len(*sortedRevisions) - limit - 2 + if exceedNum <= 0 { + return sortedRevisions, nil + } + + for _, revision := range *sortedRevisions { + if exceedNum == 0 { + break + } + + if rm.ownerGetter.IsInUsed(cd, revision.Name) { + continue + } + + if err := rm.Delete(context.TODO(), revision); err != nil { + return sortedRevisions, err + } + + exceedNum-- + } + cleanedRevisions := (*sortedRevisions)[exceedNum:] + + return &cleanedRevisions, nil +} + +func (rm *RevisionManager) createControllerRevision(ctx context.Context, parent metav1.Object, revision *apps.ControllerRevision, collisionCount *int32) (*apps.ControllerRevision, error) { + if collisionCount == nil { + return nil, fmt.Errorf("collisionCount should not be nil") + } + + // Clone the input + clone := revision.DeepCopy() + + var err error + // Continue to attempt to create the revision updating the name with a new hash on each iteration + for { + hash := hashControllerRevision(revision, collisionCount) + // Update the revisions name + clone.Name = controllerRevisionName(parent.GetName(), hash) + err = rm.Create(ctx, clone) + if errors.IsAlreadyExists(err) { + exists := &apps.ControllerRevision{} + err := rm.Get(ctx, types.NamespacedName{Namespace: clone.Namespace, Name: clone.Name}, exists) + if err != nil { + return nil, err + } + if bytes.Equal(exists.Data.Raw, clone.Data.Raw) { + return exists, nil + } + *collisionCount++ + continue + } + return clone, err + } +} + +// controllerRevisionName returns the Name for a ControllerRevision in the form prefix-hash. If the length +// of prefix is greater than 223 bytes, it is truncated to allow for a name that is no larger than 253 bytes. +func controllerRevisionName(prefix string, hash string) string { + if len(prefix) > 223 { + prefix = prefix[:223] + } + + return fmt.Sprintf("%s-%s", prefix, hash) +} + +// hashControllerRevision hashes the contents of revision's Data using FNV hashing. If probe is not nil, the byte value +// of probe is added written to the hash as well. The returned hash will be a safe encoded string to avoid bad words. +func hashControllerRevision(revision *apps.ControllerRevision, probe *int32) string { + hf := fnv.New32() + if len(revision.Data.Raw) > 0 { + hf.Write(revision.Data.Raw) + } + if revision.Data.Object != nil { + DeepHashObject(hf, revision.Data.Object) + } + if probe != nil { + hf.Write([]byte(strconv.FormatInt(int64(*probe), 10))) + } + return rand.SafeEncodeString(fmt.Sprint(hf.Sum32())) +} + +// newRevision creates a new ControllerRevision containing a patch that reapplies the target state of set. +// The Revision of the returned ControllerRevision is set to revision. If the returned error is nil, the returned +// ControllerRevision is valid. StatefulSet revisions are stored as patches that re-apply the current state of set +// to a new StatefulSet using a strategic merge patch to replace the saved state of the new StatefulSet. +func (rm *RevisionManager) newRevision(set metav1.Object, revision int64, collisionCount *int32) (*apps.ControllerRevision, error) { + patch, err := rm.ownerGetter.GetPatch(set) + if err != nil { + return nil, err + } + + runtimeObj, ok := set.(runtime.Object) + if !ok { + return nil, fmt.Errorf("revision owner %s/%s does not implement runtime Object interface", set.GetNamespace(), set.GetName()) + } + gvk, err := apiutil.GVKForObject(runtimeObj, rm.scheme) + if err != nil { + return nil, err + } + + revisionLabels := rm.ownerGetter.GetSelectorLabels(set) + if revisionLabels == nil { + revisionLabels = map[string]string{} + } + + if selector := rm.ownerGetter.GetSelector(set); selector != nil { + for k, v := range selector.MatchLabels { + revisionLabels[k] = v + } + } + + cr, err := newControllerRevision(set, + gvk, + revisionLabels, + runtime.RawExtension{Raw: patch}, + revision, + collisionCount) + if err != nil { + return nil, err + } + + cr.Namespace = set.GetNamespace() + if cr.ObjectMeta.Annotations == nil { + cr.ObjectMeta.Annotations = make(map[string]string) + } + for key, value := range set.GetAnnotations() { + cr.ObjectMeta.Annotations[key] = value + } + + if cr.ObjectMeta.Labels == nil { + cr.ObjectMeta.Labels = make(map[string]string) + } + + return cr, nil +} + +// nextRevision finds the next valid revision number based on revisions. If the length of revisions +// is 0 this is 1. Otherwise, it is 1 greater than the largest revision's Revision. This method +// assumes that revisions has been sorted by Revision. +func nextRevision(revisions []*apps.ControllerRevision) int64 { + count := len(revisions) + if count <= 0 { + return 1 + } + return revisions[count-1].Revision + 1 +} + +// SortControllerRevisions sorts revisions by their Revision. +func SortControllerRevisions(revisions []*apps.ControllerRevision) { + sort.Sort(byRevision(revisions)) +} + +// byRevision implements sort.Interface to allow ControllerRevisions to be sorted by Revision. +type byRevision []*apps.ControllerRevision + +func (br byRevision) Len() int { + return len(br) +} + +func (br byRevision) Less(i, j int) bool { + return br[i].Revision < br[j].Revision +} + +func (br byRevision) Swap(i, j int) { + br[i], br[j] = br[j], br[i] +} + +// EqualRevision returns true if lhs and rhs are either both nil, or both have same labels and annotations, or bath point +// to non-nil ControllerRevisions that contain semantically equivalent data. Otherwise this method returns false. +func EqualRevision(lhs *apps.ControllerRevision, rhs *apps.ControllerRevision) bool { + var lhsHash, rhsHash *uint32 + if lhs == nil || rhs == nil { + return lhs == rhs + } + + if hs, found := lhs.Labels[ControllerRevisionHashLabel]; found { + hash, err := strconv.ParseInt(hs, 10, 32) + if err == nil { + lhsHash = new(uint32) + *lhsHash = uint32(hash) + } + } + if hs, found := rhs.Labels[ControllerRevisionHashLabel]; found { + hash, err := strconv.ParseInt(hs, 10, 32) + if err == nil { + rhsHash = new(uint32) + *rhsHash = uint32(hash) + } + } + if lhsHash != nil && rhsHash != nil && *lhsHash != *rhsHash { + return false + } + return bytes.Equal(lhs.Data.Raw, rhs.Data.Raw) && apiequality.Semantic.DeepEqual(lhs.Data.Object, rhs.Data.Object) +} + +// FindEqualRevisions returns all ControllerRevisions in revisions that are equal to needle using EqualRevision as the +// equality test. The returned slice preserves the order of revisions. +func FindEqualRevisions(revisions []*apps.ControllerRevision, needle *apps.ControllerRevision) []*apps.ControllerRevision { + var eq []*apps.ControllerRevision + for i := range revisions { + if EqualRevision(revisions[i], needle) { + eq = append(eq, revisions[i]) + } + } + return eq +} + +// newControllerRevision returns a ControllerRevision with a ControllerRef pointing to parent and indicating that +// parent is of parentKind. The ControllerRevision has labels matching template labels, contains Data equal to data, and +// has a Revision equal to revision. The collisionCount is used when creating the name of the ControllerRevision +// so the name is likely unique. If the returned error is nil, the returned ControllerRevision is valid. If the +// returned error is not nil, the returned ControllerRevision is invalid for use. +func newControllerRevision(parent metav1.Object, + parentKind schema.GroupVersionKind, + templateLabels map[string]string, + data runtime.RawExtension, + revision int64, + collisionCount *int32) (*apps.ControllerRevision, error) { + labelMap := make(map[string]string) + for k, v := range templateLabels { + labelMap[k] = v + } + blockOwnerDeletion := true + isController := true + cr := &apps.ControllerRevision{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labelMap, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: parentKind.GroupVersion().String(), + Kind: parentKind.Kind, + Name: parent.GetName(), + UID: parent.GetUID(), + BlockOwnerDeletion: &blockOwnerDeletion, + Controller: &isController, + }, + }, + }, + Data: data, + Revision: revision, + } + hash := hashControllerRevision(cr, collisionCount) + cr.Name = controllerRevisionName(parent.GetName(), hash) + cr.Labels[ControllerRevisionHashLabel] = hash + return cr, nil +} diff --git a/pkg/controllers/utils/slow_start.go b/pkg/controllers/utils/slow_start.go new file mode 100644 index 00000000..085aae58 --- /dev/null +++ b/pkg/controllers/utils/slow_start.go @@ -0,0 +1,75 @@ +/* +Copyright 2016 The Kubernetes Authors. +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import "sync" + +const ( + SlowStartInitialBatchSize = 1 +) + +func intMin(l, r int) int { + if l < r { + return l + } + + return r +} + +// SlowStartBatch tries to call the provided function a total of 'count' times, +// starting slow to check for errors, then speeding up if calls succeed. +// +// It groups the calls into batches, starting with a group of initialBatchSize. +// Within each batch, it may call the function multiple times concurrently. +// +// If a whole batch succeeds, the next batch may get exponentially larger. +// If there are any failures in a batch, all remaining batches are skipped +// after waiting for the current batch to complete. +// +// It returns the number of successful calls to the function. +func SlowStartBatch(count int, initialBatchSize int, shortCircuit bool, fn func(int, error) error) (int, error) { + remaining := count + successes := 0 + index := 0 + var gotErr error + for batchSize := intMin(remaining, initialBatchSize); batchSize > 0; batchSize = intMin(2*batchSize, remaining) { + errCh := make(chan error, batchSize) + var wg sync.WaitGroup + wg.Add(batchSize) + for i := 0; i < batchSize; i++ { + go func(index int) { + defer wg.Done() + if err := fn(index, gotErr); err != nil { + errCh <- err + } + }(index) + index++ + } + wg.Wait() + curSuccesses := batchSize - len(errCh) + successes += curSuccesses + if len(errCh) > 0 { + gotErr = <-errCh + if shortCircuit { + return successes, gotErr + } + } + remaining -= batchSize + } + return successes, gotErr +} diff --git a/pkg/utils/inject/inject.go b/pkg/utils/inject/inject.go new file mode 100644 index 00000000..6e350a33 --- /dev/null +++ b/pkg/utils/inject/inject.go @@ -0,0 +1,60 @@ +/* + * + * Copyright 2023 The KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +package inject + +import ( + "context" + + appv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + FieldIndexOwnerRefUID = "ownerRefUID" +) + +func NewCacheWithFieldIndex(config *rest.Config, opts cache.Options) (cache.Cache, error) { + c, err := cache.New(config, opts) + if err != nil { + return c, err + } + + c.IndexField(context.TODO(), &corev1.Pod{}, FieldIndexOwnerRefUID, func(pod client.Object) []string { + ownerRef := metav1.GetControllerOf(pod) + if ownerRef == nil { + return nil + } + + return []string{string(ownerRef.UID)} + }) + + c.IndexField(context.TODO(), &appv1.ControllerRevision{}, FieldIndexOwnerRefUID, func(revision client.Object) []string { + ownerRef := metav1.GetControllerOf(revision) + if ownerRef == nil { + return nil + } + + return []string{string(ownerRef.UID)} + }) + return c, err +}