From ef085726e6d59a82277c27a38fa2164c601c90b8 Mon Sep 17 00:00:00 2001 From: Brian Groux Date: Fri, 22 Oct 2021 14:20:19 -0400 Subject: [PATCH 1/2] Add finished state change and clean up post execution --- api/v1alpha1/k6_types.go | 12 ++++--- api/v1alpha1/zz_generated.deepcopy.go | 1 + config/crd/bases/k6.io_k6s.yaml | 7 ++++ controllers/k6_controller.go | 9 +++-- controllers/k6_reconcile.go | 50 +++++++++++++++++++++++++++ 5 files changed, 72 insertions(+), 7 deletions(-) create mode 100644 controllers/k6_reconcile.go diff --git a/api/v1alpha1/k6_types.go b/api/v1alpha1/k6_types.go index 30f78077..9d67c78a 100644 --- a/api/v1alpha1/k6_types.go +++ b/api/v1alpha1/k6_types.go @@ -60,7 +60,7 @@ type K6Spec struct { Quiet string `json:"quiet,omitempty"` Paused string `json:"paused,omitempty"` Scuttle K6Scuttle `json:"scuttle,omitempty"` - // Cleanup Cleanup `json:"cleanup,omitempty"` // TODO + Cleanup Cleanup `json:"cleanup,omitempty"` } // K6Script describes where the script to execute the tests is found @@ -81,12 +81,14 @@ type K6Configmap struct { File string `json:"file,omitempty"` } -// Cleanup allows for automatic cleanup of resources pre or post execution -// +kubebuilder:validation:Enum=pre;post -// type Cleanup string +//TODO: cleanup pre-execution? + +// Cleanup allows for automatic cleanup of resources post execution +// +kubebuilder:validation:Enum=post +type Cleanup string // Stage describes which stage of the test execution lifecycle our runners are in -// +kubebuilder:validation:Enum=created;started +// +kubebuilder:validation:Enum=created;started;finished type Stage string // K6Status defines the observed state of K6 diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index bb28003f..2a1bfcec 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/config/crd/bases/k6.io_k6s.yaml b/config/crd/bases/k6.io_k6s.yaml index 7eaa32b6..494ec871 100644 --- a/config/crd/bases/k6.io_k6s.yaml +++ b/config/crd/bases/k6.io_k6s.yaml @@ -38,6 +38,12 @@ spec: properties: arguments: type: string + cleanup: + description: Cleanup allows for automatic cleanup of resources post + execution + enum: + - post + type: string parallelism: format: int32 type: integer @@ -1692,6 +1698,7 @@ spec: enum: - created - started + - finished type: string type: object type: object diff --git a/controllers/k6_controller.go b/controllers/k6_controller.go index d4d11937..c6b0758a 100644 --- a/controllers/k6_controller.go +++ b/controllers/k6_controller.go @@ -20,6 +20,7 @@ import ( "github.com/go-logr/logr" "github.com/grafana/k6-operator/api/v1alpha1" + batchv1 "k8s.io/api/batch/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" @@ -61,10 +62,13 @@ func (r *K6Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { case "created": return StartJobs(ctx, log, k6, r) case "started": - // wait for test to finish and then mark as finished - return ctrl.Result{}, nil + return ReconcileJobs(ctx, log, k6, r) case "finished": // delete if configured + if k6.Spec.Cleanup == "post" { + log.Info("Cleaning up all resources") + r.Delete(ctx, k6) + } // notify if configured return ctrl.Result{}, nil } @@ -78,5 +82,6 @@ func (r *K6Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { func (r *K6Reconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.K6{}). + Owns(&batchv1.Job{}). Complete(r) } diff --git a/controllers/k6_reconcile.go b/controllers/k6_reconcile.go new file mode 100644 index 00000000..2fdff42f --- /dev/null +++ b/controllers/k6_reconcile.go @@ -0,0 +1,50 @@ +package controllers + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + "github.com/grafana/k6-operator/api/v1alpha1" + batchv1 "k8s.io/api/batch/v1" + "k8s.io/apimachinery/pkg/labels" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Reconcile k6 status with job status +func ReconcileJobs(ctx context.Context, log logr.Logger, k6 *v1alpha1.K6, r *K6Reconciler) (ctrl.Result, error) { + selector := labels.SelectorFromSet(map[string]string{ + "app": "k6", + "k6_cr": k6.Name, + }) + + opts := &client.ListOptions{LabelSelector: selector, Namespace: k6.Namespace} + jl := &batchv1.JobList{} + + if err := r.List(ctx, jl, opts); err != nil { + log.Error(err, "Could not list jobs") + return ctrl.Result{}, err + } + + //TODO: We should distinguish between suceeded/failed + var finished int32 + for _, job := range jl.Items { + if job.Status.Active != 0 { + continue + } + finished++ + } + + log.Info(fmt.Sprintf("%d/%d jobs complete", finished, k6.Spec.Parallelism+1)) + + // parallelism (pods) + starter pod = total expected + if finished == k6.Spec.Parallelism+1 { + k6.Status.Stage = "finished" + if err := r.Client.Status().Update(ctx, k6); err != nil { + log.Error(err, "Could not update status of custom resource") + } + } + + return ctrl.Result{}, nil +} From 012c0bf17c90e571ebe079f099ea0f27bf7d1bd7 Mon Sep 17 00:00:00 2001 From: Brian Groux Date: Wed, 9 Feb 2022 10:06:21 -0500 Subject: [PATCH 2/2] Update reconcile->finish naming, inline with create and start --- controllers/k6_controller.go | 2 +- controllers/{k6_reconcile.go => k6_finish.go} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename controllers/{k6_reconcile.go => k6_finish.go} (88%) diff --git a/controllers/k6_controller.go b/controllers/k6_controller.go index c6b0758a..d6641550 100644 --- a/controllers/k6_controller.go +++ b/controllers/k6_controller.go @@ -62,7 +62,7 @@ func (r *K6Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { case "created": return StartJobs(ctx, log, k6, r) case "started": - return ReconcileJobs(ctx, log, k6, r) + return FinishJobs(ctx, log, k6, r) case "finished": // delete if configured if k6.Spec.Cleanup == "post" { diff --git a/controllers/k6_reconcile.go b/controllers/k6_finish.go similarity index 88% rename from controllers/k6_reconcile.go rename to controllers/k6_finish.go index 2fdff42f..b7fb9550 100644 --- a/controllers/k6_reconcile.go +++ b/controllers/k6_finish.go @@ -12,8 +12,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// Reconcile k6 status with job status -func ReconcileJobs(ctx context.Context, log logr.Logger, k6 *v1alpha1.K6, r *K6Reconciler) (ctrl.Result, error) { +// Mark k6 as finished as jobs finish +func FinishJobs(ctx context.Context, log logr.Logger, k6 *v1alpha1.K6, r *K6Reconciler) (ctrl.Result, error) { selector := labels.SelectorFromSet(map[string]string{ "app": "k6", "k6_cr": k6.Name,