test(e2e): extended e2e test scenarios

.chainsaw.yaml

@@ -5,11 +5,11 @@ metadata:
 spec:
   timeouts:
     apply: 45s
-    assert: 300s
+    assert: 600s
     cleanup: 45s
     delete: 25s
     error: 10s
     exec: 45s
   skipDelete: false
   failFast: true
-  parallel: 4
+  parallel: 1

.gitignore

@@ -29,6 +29,9 @@ go.work
 # generated files
 dist/
 
+# E2E test report
+chainsaw-report.xml
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

api/v1alpha1/lkeclusterconfig_types.go

@@ -139,12 +139,13 @@ func (n NodePoolStatus) IsEqual(cmp NodePoolStatus) bool {
 	return n.NodePoolDetails.IsEqual(cmp.NodePoolDetails)
 }
 
-// +kubebuilder:validation:Enum=Active;Deleting;Provisioning;Unknown;Updating
+// +kubebuilder:validation:Enum=Active;Deleting;Error;Provisioning;Unknown;Updating
 type Phase string
 
 const (
 	PhaseActive       Phase = "Active"
 	PhaseDeleting     Phase = "Deleting"
+	PhaseError        Phase = "Error"
 	PhaseProvisioning Phase = "Provisioning"
 	PhaseUpdating     Phase = "Updating"
 	PhaseUnknown      Phase = "Unknown"

cmd/main.go

@@ -19,18 +19,20 @@ package main
 import (
 	"crypto/tls"
 	"flag"
+	"net/http"
 	"os"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
-	"k8s.io/client-go/kubernetes"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
 	"github.com/anza-labs/lke-operator/internal/controller"
 	tracedk8s "github.com/anza-labs/lke-operator/internal/k8s/traced"
 	"github.com/anza-labs/lke-operator/internal/version"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -144,10 +146,15 @@ func main() {
 		os.Exit(1)
 	}
 
+	rest := ctrl.GetConfigOrDie()
+	rest.WrapTransport = func(rt http.RoundTripper) http.RoundTripper {
+		return otelhttp.NewTransport(rt)
+	}
+
 	if err = (&controller.LKEClusterConfigReconciler{
 		Client:           tracedk8s.NewClientWithTracing(mgr.GetClient(), "main_mgr_client"),
 		Scheme:           mgr.GetScheme(),
-		KubernetesClient: kubernetes.NewForConfigOrDie(ctrl.GetConfigOrDie()),
+		KubernetesClient: kubernetes.NewForConfigOrDie(rest),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller",
 			"controller", "LKEClusterConfig")

config/crd/bases/lke.anza-labs.dev_lkeclusterconfigs.yaml

@@ -185,6 +185,7 @@ spec:
                 enum:
                 - Active
                 - Deleting
+                - Error
                 - Provisioning
                 - Unknown
                 - Updating

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/go-logr/logr v1.4.2
 	github.com/go-resty/resty/v2 v2.13.1
 	github.com/linode/linodego v1.34.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0
 	go.opentelemetry.io/otel v1.27.0
 	go.opentelemetry.io/otel/trace v1.27.0
 	k8s.io/api v0.30.1
@@ -265,7 +266,6 @@ require (
 	go.etcd.io/etcd/client/v3 v3.5.11 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.23.1 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.23.1 // indirect
 	go.opentelemetry.io/otel/metric v1.27.0 // indirect

internal/controller/lkeclusterconfig.go

@@ -96,8 +96,8 @@ func (r *LKEClusterConfigReconciler) onChangeCreate(
 	}
 
 	if lke.Spec.HighAvailability != nil {
-		opts.ControlPlane = &linodego.LKEClusterControlPlane{
-			HighAvailability: *lke.Spec.HighAvailability,
+		opts.ControlPlane = &linodego.LKEClusterControlPlaneOptions{
+			HighAvailability: lke.Spec.HighAvailability,
 		}
 	}
 
@@ -394,8 +394,8 @@ func updateControlPlane(
 		lke.Spec.HighAvailability = mkptr(false)
 	}
 
-	opts.ControlPlane = &linodego.LKEClusterControlPlane{
-		HighAvailability: *lke.Spec.HighAvailability,
+	opts.ControlPlane = &linodego.LKEClusterControlPlaneOptions{
+		HighAvailability: lke.Spec.HighAvailability,
 	}
 
 	return opts, cluster.ControlPlane.HighAvailability != *lke.Spec.HighAvailability

internal/controller/lkeclusterconfig_controller.go

@@ -55,7 +55,7 @@ type LKEClusterConfigReconciler struct {
 // For more details, check Reconcile and its Result here:
 // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.17.3/pkg/reconcile
 func (r *LKEClusterConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
-	log := log.FromContext(ctx).WithName("reconcile").WithValues("object.namespaced_name", req)
+	log := log.FromContext(ctx, "object.namespaced_name", req)
 
 	log.Info("reconciling")
 
@@ -128,6 +128,7 @@ func (r *LKEClusterConfigReconciler) setFailureMessage(
 	err error,
 ) error {
 	lke.Status.FailureMessage = mkptr(err.Error())
+	lke.Status.Phase = mkptr(lkev1alpha1.PhaseError)
 	if uerr := r.Update(ctx, lke); uerr != nil {
 		return errors.Join(err, uerr)
 	}

internal/controller/lkeclusterconfig_test.go

@@ -200,7 +200,7 @@ func Test_updateControlPlane(t *testing.T) {
 					tc.expectedChange, change)
 			}
 
-			if opts.ControlPlane.HighAvailability != tc.expectedHA {
+			if opts.ControlPlane.HighAvailability == nil || *opts.ControlPlane.HighAvailability != tc.expectedHA {
 				t.Errorf("expected HA value: %#+v, got: %#+v",
 					tc.expectedHA, opts.ControlPlane.HighAvailability)
 			}

test/base/apply.yaml → test/00-minimal/apply.yaml

@@ -1,17 +1,17 @@
 apiVersion: lke.anza-labs.dev/v1alpha1
 kind: LKEClusterConfig
 metadata:
-  name: base
+  name: minimal
   annotations:
     lke.anza-labs.dev/tags: |
-      test-base
-      chainsaw
+      test-minimal
+      chainsaw-e2e
 spec:
   tokenSecretRef:
     name: test-token
     namespace: test-shared
   region: eu-central
   nodePools:
-    base:
+    np-1:
       nodeCount: 3
       linodeType: g6-standard-1

test/00-minimal/assert.yaml

@@ -0,0 +1,24 @@
+apiVersion: lke.anza-labs.dev/v1alpha1
+kind: LKEClusterConfig
+metadata:
+  name: minimal
+  annotations:
+    lke.anza-labs.dev/tags: |
+      test-minimal
+      chainsaw-e2e
+spec:
+  tokenSecretRef:
+    name: test-token
+    namespace: test-shared
+  region: eu-central
+  nodePools:
+    np-1:
+      nodeCount: 3
+      linodeType: g6-standard-1
+status:
+  phase: Active
+  nodePoolStatuses:
+    np-1:
+      details:
+        nodeCount: 3
+        linodeType: g6-standard-1

test/base/chainsaw-test.yaml → test/00-minimal/chainsaw-test.yaml

@@ -1,7 +1,7 @@
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
-  name: base
+  name: minimal
 spec:
   steps:
   - try:

test/01-multiple-node-pools/apply.yaml

@@ -0,0 +1,23 @@
+apiVersion: lke.anza-labs.dev/v1alpha1
+kind: LKEClusterConfig
+metadata:
+  name: multi-np
+  annotations:
+    lke.anza-labs.dev/tags: |
+      test-multi-np
+      chainsaw-e2e
+spec:
+  tokenSecretRef:
+    name: test-token
+    namespace: test-shared
+  region: eu-central
+  nodePools:
+    np-1:
+      nodeCount: 3
+      linodeType: g6-standard-1
+    np-2:
+      nodeCount: 3
+      linodeType: g6-standard-1
+    np-3:
+      nodeCount: 3
+      linodeType: g6-standard-1

test/01-multiple-node-pools/assert.yaml

@@ -0,0 +1,38 @@
+apiVersion: lke.anza-labs.dev/v1alpha1
+kind: LKEClusterConfig
+metadata:
+  name: multi-np
+  annotations:
+    lke.anza-labs.dev/tags: |
+      test-multi-np
+      chainsaw-e2e
+spec:
+  tokenSecretRef:
+    name: test-token
+    namespace: test-shared
+  region: eu-central
+  nodePools:
+    np-1:
+      nodeCount: 3
+      linodeType: g6-standard-1
+    np-2:
+      nodeCount: 3
+      linodeType: g6-standard-1
+    np-3:
+      nodeCount: 3
+      linodeType: g6-standard-1
+status:
+  phase: Active
+  nodePoolStatuses:
+    np-1:
+      details:
+        nodeCount: 3
+        linodeType: g6-standard-1
+    np-2:
+      details:
+        nodeCount: 3
+        linodeType: g6-standard-1
+    np-3:
+      details:
+        nodeCount: 3
+        linodeType: g6-standard-1

test/01-multiple-node-pools/chainsaw-test.yaml

@@ -0,0 +1,11 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: multiple-node-pools
+spec:
+  steps:
+  - try:
+    - apply:
+        file: ./apply.yaml
+    - assert:
+        file: ./assert.yaml

test/02-autoscaler/apply.yaml

@@ -0,0 +1,20 @@
+apiVersion: lke.anza-labs.dev/v1alpha1
+kind: LKEClusterConfig
+metadata:
+  name: autoscaler
+  annotations:
+    lke.anza-labs.dev/tags: |
+      test-autoscaler
+      chainsaw-e2e
+spec:
+  tokenSecretRef:
+    name: test-token
+    namespace: test-shared
+  region: eu-central
+  nodePools:
+    np-1:
+      nodeCount: 1
+      linodeType: g6-standard-1
+      autoscaler:
+        min: 1
+        max: 3

test/02-autoscaler/assert.yaml

@@ -0,0 +1,30 @@
+apiVersion: lke.anza-labs.dev/v1alpha1
+kind: LKEClusterConfig
+metadata:
+  name: autoscaler
+  annotations:
+    lke.anza-labs.dev/tags: |
+      test-autoscaler
+      chainsaw-e2e
+spec:
+  tokenSecretRef:
+    name: test-token
+    namespace: test-shared
+  region: eu-central
+  nodePools:
+    np-1:
+      nodeCount: 1
+      linodeType: g6-standard-1
+      autoscaler:
+        min: 1
+        max: 3
+status:
+  phase: Active
+  nodePoolStatuses:
+    np-1:
+      details:
+        nodeCount: 1
+        linodeType: g6-standard-1
+        autoscaler:
+          min: 1
+          max: 3

test/02-autoscaler/chainsaw-test.yaml

@@ -0,0 +1,11 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: autoscaler
+spec:
+  steps:
+  - try:
+    - apply:
+        file: ./apply.yaml
+    - assert:
+        file: ./assert.yaml

test/base/assert.yaml → test/03-high-availability/apply.yaml

@@ -1,19 +1,18 @@
 apiVersion: lke.anza-labs.dev/v1alpha1
 kind: LKEClusterConfig
 metadata:
-  name: base
+  name: high-availability
   annotations:
     lke.anza-labs.dev/tags: |
-      test-base
-      chainsaw
+      test-ha
+      chainsaw-e2e
 spec:
   tokenSecretRef:
     name: test-token
     namespace: test-shared
   region: eu-central
+  highAvailability: true
   nodePools:
-    base:
+    np-1:
       nodeCount: 3
       linodeType: g6-standard-1
-status:
-  phase: Active

test/03-high-availability/assert.yaml

@@ -0,0 +1,25 @@
+apiVersion: lke.anza-labs.dev/v1alpha1
+kind: LKEClusterConfig
+metadata:
+  name: high-availability
+  annotations:
+    lke.anza-labs.dev/tags: |
+      test-ha
+      chainsaw-e2e
+spec:
+  tokenSecretRef:
+    name: test-token
+    namespace: test-shared
+  region: eu-central
+  highAvailability: true
+  nodePools:
+    np-1:
+      nodeCount: 3
+      linodeType: g6-standard-1
+status:
+  phase: Active
+  nodePoolStatuses:
+    np-1:
+      details:
+        nodeCount: 3
+        linodeType: g6-standard-1