Skip to content

Commit

Permalink
Merge pull request #405 from mlycore/update-stress
Browse files Browse the repository at this point in the history 
fix(operator): fix cpu stress and memory stress builder
  • Loading branch information
@strongduanmu
strongduanmu authored Jun 8, 2023
2 parents 857be0e + eb58e3a commit 93ddb65
Show file tree
Hide file tree
Showing 5 changed files with 381 additions and 41 deletions.
10 changes: 5 additions & 5 deletions shardingsphere-operator/api/v1alpha1/chaos_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,15 +154,15 @@ type PodChaosSpec struct {
// PodActionParams Optional parameters for pod type configuration
type PodChaosParams struct {
// +optional
PodFailure *PodFailureParams `json:"podFailure,omitempty"`
PodFailure *PodFailureParams `json:"podFailure,omitempty" yaml:"PodFailure,omitempty"`
// +optional
ContainerKill *ContainerKillParams `json:"containerKill,omitempty"`
ContainerKill *ContainerKillParams `json:"containerKill,omitempty" yaml:"containerKill,omitempty"`
//+optional
CPUStress *CPUStressParams `json:"cpuStress,omitempty"`
CPUStress *CPUStressParams `json:"cpuStress,omitempty" yaml:"cpuStress,omitempty"`
//+optional
MemoryStress *MemoryStressParams `json:"memoryStress,omitempty"`
MemoryStress *MemoryStressParams `json:"memoryStress,omitempty" yaml:"memoryStress,omitempty"`
// +optional
PodKill *PodKillParams `json:"podKill,omitempty"`
PodKill *PodKillParams `json:"podKill,omitempty" yaml:"podKill,omitempty"`
}

type PodFailureParams struct {
Expand Down
115 changes: 101 additions & 14 deletions shardingsphere-operator/pkg/controllers/chaos_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,11 +150,27 @@ func (r *ChaosReconciler) updateChaosCondition(ctx context.Context, chaos *v1alp
}

if chaos.Spec.EmbedChaos.PodChaos != nil {
pc, err := r.Chaos.GetPodChaosByNamespacedName(ctx, namespacedName)
if err != nil {
return err
switch chaos.Spec.EmbedChaos.PodChaos.Action {
case v1alpha1.CPUStress:
fallthrough
case v1alpha1.MemoryStress:
sc, err := r.Chaos.GetStressChaosByNamespacedName(ctx, namespacedName)
if err != nil {
return err
}
chaos.Status.ChaosCondition = chaosmesh.ConvertChaosStatus(ctx, chaos, sc)
case v1alpha1.PodFailure:
fallthrough
case v1alpha1.PodKill:
fallthrough
case v1alpha1.ContainerKill:
pc, err := r.Chaos.GetPodChaosByNamespacedName(ctx, namespacedName)
if err != nil {
return err
}
chaos.Status.ChaosCondition = chaosmesh.ConvertChaosStatus(ctx, chaos, pc)
}
chaos.Status.ChaosCondition = chaosmesh.ConvertChaosStatus(ctx, chaos, pc)

}

if chaos.Spec.EmbedChaos.NetworkChaos != nil {
Expand Down Expand Up @@ -205,18 +221,29 @@ func (r *ChaosReconciler) finalize(ctx context.Context, ssChaos *v1alpha1.Chaos)
func (r *ChaosReconciler) deleteExternalResources(ctx context.Context, chao *v1alpha1.Chaos) error {
nameSpacedName := types.NamespacedName{Namespace: chao.Namespace, Name: chao.Name}
if chao.Spec.EmbedChaos.PodChaos != nil {
if err := r.deletePodChaos(ctx, nameSpacedName); err != nil {
return err
switch chao.Spec.EmbedChaos.PodChaos.Action {
case v1alpha1.CPUStress:
fallthrough
case v1alpha1.MemoryStress:
if err := r.deleteStressChaos(ctx, nameSpacedName); err != nil {
return err
}
case v1alpha1.PodFailure:
fallthrough
case v1alpha1.PodKill:
fallthrough
case v1alpha1.ContainerKill:
if err := r.deletePodChaos(ctx, nameSpacedName); err != nil {
return err
}
}

return nil
}

if chao.Spec.EmbedChaos.NetworkChaos != nil {
if err := r.deleteNetworkChaos(ctx, nameSpacedName); err != nil {
return err
}

return nil
}

Expand Down Expand Up @@ -265,16 +292,50 @@ func (r *ChaosReconciler) deleteNetworkChaos(ctx context.Context, namespacedName
return nil
}

func (r *ChaosReconciler) reconcilePodChaos(ctx context.Context, chaos *v1alpha1.Chaos, namespacedName types.NamespacedName) error {
pc, err := r.getPodChaosByNamespacedName(ctx, namespacedName)
func (r *ChaosReconciler) deleteStressChaos(ctx context.Context, namespacedName types.NamespacedName) error {
sc, err := r.getStressChaosByNamespacedName(ctx, namespacedName)
if err != nil {
return err
}
if pc != nil {
return r.updatePodChaos(ctx, chaos, pc)
if sc != nil {
if err := r.Chaos.DeleteStressChaos(ctx, sc); err != nil {
return err
}
}

return r.createPodChaos(ctx, chaos)
return nil
}

func (r *ChaosReconciler) reconcilePodChaos(ctx context.Context, chaos *v1alpha1.Chaos, namespacedName types.NamespacedName) error {
switch chaos.Spec.EmbedChaos.PodChaos.Action {
case v1alpha1.PodFailure:
fallthrough
case v1alpha1.ContainerKill:
fallthrough
case v1alpha1.PodKill:
pc, err := r.getPodChaosByNamespacedName(ctx, namespacedName)
if err != nil {
return err
}
if pc != nil {
return r.updatePodChaos(ctx, chaos, pc)
}

return r.createPodChaos(ctx, chaos)
case v1alpha1.CPUStress:
fallthrough
case v1alpha1.MemoryStress:
sc, err := r.getStressChaosByNamespacedName(ctx, namespacedName)
if err != nil {
return err
}
if sc != nil {
return r.updateStressChaos(ctx, chaos, sc)
}

return r.createStressChaos(ctx, chaos)
}
return nil
}

func (r *ChaosReconciler) getPodChaosByNamespacedName(ctx context.Context, namespacedName types.NamespacedName) (chaosmesh.PodChaos, error) {
Expand Down Expand Up @@ -330,7 +391,7 @@ func (r *ChaosReconciler) createNetworkChaos(ctx context.Context, chaos *v1alpha
return err
}

r.Events.Event(chaos, "Normal", "created", fmt.Sprintf("networkChaos %s", " is created successfully"))
r.Events.Event(chaos, "Normal", "created", fmt.Sprintf("NetworkChaos %s", " is created successfully"))
return nil
}

Expand All @@ -342,6 +403,32 @@ func (r *ChaosReconciler) getNetworkChaosByNamespacedName(ctx context.Context, n
return nc, nil
}

func (r *ChaosReconciler) getStressChaosByNamespacedName(ctx context.Context, namespacedName types.NamespacedName) (chaosmesh.StressChaos, error) {
pc, err := r.Chaos.GetStressChaosByNamespacedName(ctx, namespacedName)
if err != nil {
return nil, err
}
return pc, nil
}

func (r *ChaosReconciler) createStressChaos(ctx context.Context, chaos *v1alpha1.Chaos) error {
err := r.Chaos.CreateStressChaos(ctx, chaos)
if err != nil {
return err
}
r.Events.Event(chaos, "Normal", "Created", fmt.Sprintf("StressChaos %s", " is created successfully"))
return nil
}

func (r *ChaosReconciler) updateStressChaos(ctx context.Context, chaos *v1alpha1.Chaos, stress chaosmesh.StressChaos) error {
err := r.Chaos.UpdateStressChaos(ctx, stress, chaos)
if err != nil {
return err
}

return nil
}

// SetupWithManager sets up the controller with the Manager.
func (r *ChaosReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
Expand Down
63 changes: 41 additions & 22 deletions shardingsphere-operator/pkg/kubernetes/chaosmesh/builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,21 +58,31 @@ var (

type GenericChaos interface{}

func ConvertChaosStatus(ctx context.Context, ssChaos *v1alpha1.Chaos, chaos GenericChaos) v1alpha1.ChaosCondition {
var status chaosmeshv1alpha1.ChaosStatus
func getStatus(ssChaos *v1alpha1.Chaos, chaos GenericChaos) *chaosmeshv1alpha1.ChaosStatus {
var status *chaosmeshv1alpha1.ChaosStatus
if ssChaos.Spec.EmbedChaos.PodChaos != nil {
if podChao, ok := chaos.(*chaosmeshv1alpha1.PodChaos); ok && podChao != nil {
status = *podChao.GetStatus()
} else {
return v1alpha1.Unknown
status = podChao.GetStatus()
} else if ssChao, ok := chaos.(*chaosmeshv1alpha1.StressChaos); ok && ssChao != nil {
status = ssChao.GetStatus()
}
} else if ssChaos.Spec.EmbedChaos.NetworkChaos != nil {
}

if ssChaos.Spec.EmbedChaos.NetworkChaos != nil {
if networkChaos, ok := chaos.(*chaosmeshv1alpha1.NetworkChaos); ok && networkChaos != nil {
status = *networkChaos.GetStatus()
} else {
return v1alpha1.Unknown
status = networkChaos.GetStatus()
}
}

return status
}

func ConvertChaosStatus(ctx context.Context, ssChaos *v1alpha1.Chaos, chaos GenericChaos) v1alpha1.ChaosCondition {
status := getStatus(ssChaos, chaos)
if status == nil {
return v1alpha1.Unknown
}

var conditions = map[chaosmeshv1alpha1.ChaosConditionType]bool{}
for i := range status.Conditions {
conditions[status.Conditions[i].Type] = status.Conditions[i].Status == corev1.ConditionTrue
Expand Down Expand Up @@ -105,10 +115,6 @@ func judgeCondition(condition map[chaosmeshv1alpha1.ChaosConditionType]bool, pha

func NewPodChaos(ssChao *v1alpha1.Chaos) (PodChaos, error) {
chao := ssChao.Spec.PodChaos
if chao.Action == v1alpha1.MemoryStress || chao.Action == v1alpha1.CPUStress {
return NewStressChaos(ssChao)
}

pcb := NewPodChaosBuilder()
pcb.SetName(ssChao.Name).SetNamespace(ssChao.Namespace).SetLabels(ssChao.Labels)
pcb.SetAction(string(chao.Action))
Expand Down Expand Up @@ -148,7 +154,7 @@ func NewPodChaos(ssChao *v1alpha1.Chaos) (PodChaos, error) {
return podChao, nil
}

func NewStressChaos(chaos *v1alpha1.Chaos) (PodChaos, error) {
func NewStressChaos(chaos *v1alpha1.Chaos) (StressChaos, error) {
sc := &chaosmeshv1alpha1.StressChaos{}
sc.Namespace = chaos.Namespace
sc.Name = chaos.Name
Expand All @@ -157,7 +163,6 @@ func NewStressChaos(chaos *v1alpha1.Chaos) (PodChaos, error) {
chao := chaos.Spec.PodChaos

psb := NewPodSelectorBuilder()

psb.SetNamespaces(chao.Namespaces).
SetExpressionSelectors(chao.ExpressionSelectors).
SetNodes(chao.Nodes).
Expand All @@ -166,8 +171,8 @@ func NewStressChaos(chaos *v1alpha1.Chaos) (PodChaos, error) {
SetLabelSelector(chao.LabelSelectors).
SetPods(chao.Pods)

psb.SetSelectMode(chaos.Annotations[AnnoTargetPodSelectMode]).
SetValue(chaos.Annotations[AnnoTargetPodSelectValue])
psb.SetSelectMode(chaos.Annotations[AnnoPodSelectorMode]).
SetValue(chaos.Annotations[AnnoPodSelectorValue])

sc.Spec.ContainerSelector = chaosmeshv1alpha1.ContainerSelector{
PodSelector: *psb.Build(),
Expand All @@ -187,20 +192,32 @@ func NewStressChaos(chaos *v1alpha1.Chaos) (PodChaos, error) {
}

func setCPUStressParams(sschaos *v1alpha1.Chaos, chaos *chaosmeshv1alpha1.StressChaos) {
cpu := chaosmeshv1alpha1.CPUStressor{
cpu := &chaosmeshv1alpha1.CPUStressor{
Stressor: chaosmeshv1alpha1.Stressor{
Workers: sschaos.Spec.PodChaos.Params.CPUStress.Cores,
},
Load: &sschaos.Spec.PodChaos.Params.CPUStress.Load,
}

chaos.Spec.Stressors.CPUStressor = &cpu
chaos.Spec.Stressors = &chaosmeshv1alpha1.Stressors{
CPUStressor: cpu,
}
chaos.Spec.Duration = &sschaos.Spec.PodChaos.Params.CPUStress.Duration
}

func setMemoryStressParams(sschaos *v1alpha1.Chaos, chaos *chaosmeshv1alpha1.StressChaos) error {
oom, err := strconv.Atoi(sschaos.Annotations[AnnoOOMScoreAdj])
memory := chaosmeshv1alpha1.MemoryStressor{
var (
oom int
err error
)
if adj, ok := sschaos.Annotations[AnnoOOMScoreAdj]; ok {
oom, err = strconv.Atoi(adj)
if err != nil {
return err
}
}

memory := &chaosmeshv1alpha1.MemoryStressor{
Stressor: chaosmeshv1alpha1.Stressor{
Workers: sschaos.Spec.PodChaos.Params.MemoryStress.Workers,
},
Expand All @@ -211,7 +228,9 @@ func setMemoryStressParams(sschaos *v1alpha1.Chaos, chaos *chaosmeshv1alpha1.Str
},
}

chaos.Spec.Stressors.MemoryStressor = &memory
chaos.Spec.Stressors = &chaosmeshv1alpha1.Stressors{
MemoryStressor: memory,
}
chaos.Spec.Duration = &sschaos.Spec.PodChaos.Params.MemoryStress.Duration

return err
Expand Down
Loading

0 comments on commit 93ddb65

Please sign in to comment.