Add label filtering to CTS (#28)

* Add label filtering to CTS

* Bunch of fixes

* Generate CRD

* Remove unnecessary pointer, rename toList

* Doc changes after review

* Refactor definition fetching

* Get rid of uid package from tests

* Document matchLabelExpressions

* Fix definition fetcher tests

* Add integration test for selective testing

* Typo

* Update docs/crd-cluster-test-suite.md

Co-Authored-By: Klaudia Grzondziel <35192450+klaudiagrz@users.noreply.github.com>

config/crds/testing_v1alpha1_clustertestsuite.yaml

@@ -53,9 +53,10 @@ spec:
               description: Decide which tests to execute. If not provided execute
                 all tests
               properties:
-                matchLabels:
-                  description: Find test definitions by it's labels. TestDefinition
-                    should have AT LEAST one label listed here to be executed.
+                matchLabelExpressions:
+                  description: 'Find test definitions by their labels. TestDefinition
+                    must match AT LEAST one expression listed here to be executed.
+                    For the complete grammar see: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels'
                   items:
                     type: string
                   type: array

config/samples/advanced/testsuite-select-by-labels.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: testing.kyma-project.io/v1alpha1
+kind: ClusterTestSuite
+metadata:
+  labels:
+    controller-tools.k8s.io: "1.0"
+  name: testsuite-selected-by-labels
+spec:
+  count: 1
+  selectors:
+    matchLabelExpressions:
+      # This example executes all not long tests for frontend and all tests for backend
+      - component=frontend,test-duration!=long
+      - component=backend
+

docs/crd-cluster-test-suite.md

@@ -31,7 +31,7 @@ This table lists all the possible parameters of a given resource together with t
 | **metadata.name** |    **YES**   | Specifies the name of the CR. |
 | **spec.selectors** | **NO** | Defines which tests should be executed. You can define tests by specifying their names or labels. Selectors are additive. If not defined, all tests from all Namespaces are executed.
 | **spec.selectors.matchNames** | **NO** | Lists TestDefinitions to execute. For every element on the list, specify **name** and **namespace** that refers to a TestDefinition. |
-| **spec.selectors.matchLabels** | **NO** | Lists labels that match labels of TestDefinitions to execute. A TestDefinition is selected if at least one label matches. This feature is not yet implemented. | 
+| **spec.selectors.matchLabelExpressions** | **NO** | Lists label expressions that match labels of TestDefinitions to execute. A TestDefinition is selected if at least one label expression matches. See [this](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels) document for more details. | 
 | **spec.concurrency** | **NO** | Defines how many tests can be executed at the same time, which depends on cluster size and its load. The default value is `1`.
 | **spec.suiteTimeout** | **NO** | Defines the maximal suite duration after which test executions are interrupted and marked as **Failed**. The default value is one hour. This feature is not yet implemented. 
 | **spec.count** | **NO** | Defines how many times every test should be executed. **Spec.Count** and **Spec.MaxRetries** are mutually exclusive. The default value is `1`.  

pkg/apis/testing/v1alpha1/testsuite_types.go

@@ -108,9 +108,10 @@ type TestSuiteSpec struct {
 type TestsSelector struct {
 	// Find test definitions by it's name
 	MatchNames []TestDefReference `json:"matchNames,omitempty"`
-	// Find test definitions by it's labels.
-	// TestDefinition should have AT LEAST one label listed here to be executed.
-	MatchLabels []string `json:"matchLabels,omitempty"`
+	// Find test definitions by their labels.
+	// TestDefinition must match AT LEAST one expression listed here to be executed.
+	// For the complete grammar see: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels
+	MatchLabelExpressions []string `json:"matchLabelExpressions,omitempty"`
 }
 
 type TestDefReference struct {
@@ -157,3 +158,7 @@ type TestExecution struct {
 func init() {
 	SchemeBuilder.Register(&ClusterTestSuite{}, &ClusterTestSuiteList{})
 }
+
+func (in ClusterTestSuite) HasSelector() bool {
+	return len(in.Spec.Selectors.MatchNames) > 0 || len(in.Spec.Selectors.MatchLabelExpressions) > 0
+}

pkg/controller/testsuite/testsuite_controller_test.go

@@ -330,6 +330,91 @@ func TestReconcileClusterTestSuite(t *testing.T) {
 
 	})
 
+	t.Run("selective testing", func(t *testing.T) {
+		// GIVEN
+		// Setup the Manager and Controller
+		mgr, err := manager.New(cfg, manager.Options{})
+		require.NoError(t, err)
+		c := mgr.GetClient()
+
+		testNs := generateTestNs()
+		ctx := context.Background()
+
+		require.NoError(t, add(mgr, newReconciler(mgr)))
+		stopMgr, mgrStopped := StartTestManager(t, mgr)
+
+		defer func() {
+			close(stopMgr)
+			mgrStopped.Wait()
+		}()
+
+		logf.SetLogger(logf.ZapLogger(false))
+
+		podReconciler, err := startMockPodController(mgr, 0)
+		require.NoError(t, err)
+
+		// WHEN
+		ns := &v1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: testNs,
+			},
+		}
+		err = c.Create(ctx, ns)
+		require.NoError(t, err)
+		defer cleanupK8sObject(ctx, c, ns)
+
+		testA := getSequentialTest("test-a", testNs)
+		err = c.Create(ctx, testA)
+		require.NoError(t, err)
+		defer cleanupK8sObject(ctx, c, testA)
+
+		testB := getSequentialTest("test-b", testNs)
+		testB.Labels = map[string]string{"test": "true"}
+		err = c.Create(ctx, testB)
+		require.NoError(t, err)
+		defer cleanupK8sObject(ctx, c, testB)
+
+		testC := getSequentialTest("test-c", testNs)
+		testC.Labels = map[string]string{"test": "false"}
+		err = c.Create(ctx, testC)
+		require.NoError(t, err)
+		defer cleanupK8sObject(ctx, c, testC)
+
+		suite := &testingv1alpha1.ClusterTestSuite{
+			ObjectMeta: metav1.ObjectMeta{Name: "suite-selective"},
+			Spec: testingv1alpha1.TestSuiteSpec{
+				Concurrency: 1,
+				Count:       1,
+				Selectors: testingv1alpha1.TestsSelector{
+					MatchNames: []testingv1alpha1.TestDefReference{
+						{
+							Name:      "test-a",
+							Namespace: testNs,
+						},
+					},
+					MatchLabelExpressions: []string{
+						"test=true",
+					},
+				},
+			},
+		}
+		err = c.Create(ctx, suite)
+		require.NoError(t, err)
+		defer cleanupK8sObject(ctx, c, suite)
+
+		// THEN
+		repeat.FuncAtMost(t, func() error {
+			return checkIfsuiteIsSucceeded(ctx, c, "suite-selective")
+		}, defaultAssertionTimeout)
+
+		repeat.FuncAtMost(t, func() error {
+			return checkIfPodsWereCreated(ctx, c, testNs, []string{
+				"oct-tp-suite-selective-test-a-0",
+				"oct-tp-suite-selective-test-b-0"})
+		}, defaultAssertionTimeout)
+
+		assertThatPodsCreatedSequentially(t, podReconciler.getAppliedChanges())
+	})
 }
 
 func assertThatPodsCreatedConcurrently(t *testing.T, appliedChanges []podStatusChanges) {
@@ -418,7 +503,7 @@ type mockPodReconciler struct {
 	cli client.Client
 	// mutex protecting access to applied changes slice
 	mtx sync.Mutex
-	// chronologically list of pod changes. Use getter to access it autside the struct
+	// chronologically list of pod changes. Use getter to access it outside the struct
 	appliedChanges                         []podStatusChanges
 	enforceNumberOfConcurrentlyRunningPods int
 	reconcileAfter                         time.Duration
@@ -483,8 +568,8 @@ func (r *mockPodReconciler) getLogger() logr.Logger {
 
 func startMockPodController(mgr manager.Manager, enforceConcurrentlyRunningPods int) (*mockPodReconciler, error) {
 	pr := &mockPodReconciler{
-		cli: mgr.GetClient(),
-		mtx: sync.Mutex{},
+		cli:                                    mgr.GetClient(),
+		mtx:                                    sync.Mutex{},
 		enforceNumberOfConcurrentlyRunningPods: enforceConcurrentlyRunningPods,
 		reconcileAfter:                         time.Millisecond * 100,
 	}

pkg/fetcher/definition.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"github.com/kyma-incubator/octopus/pkg/humanerr"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 
 	"github.com/kyma-incubator/octopus/pkg/apis/testing/v1alpha1"
@@ -24,19 +25,30 @@ type Definition struct {
 
 func (s *Definition) FindMatching(suite v1alpha1.ClusterTestSuite) ([]v1alpha1.TestDefinition, error) {
 	ctx := context.TODO()
-	if len(suite.Spec.Selectors.MatchNames) > 0 {
-		return s.findByNames(ctx, suite)
+
+	if suite.HasSelector() {
+		return s.findBySelector(ctx, suite)
 	}
-	// TODO(aszecowka) later so far we return all test definitions for all namespaces (https://github.com/kyma-incubator/octopus/issues/7)
-	var list v1alpha1.TestDefinitionList
-	if err := s.reader.List(ctx, &client.ListOptions{Namespace: ""}, &list); err != nil {
-		return nil, errors.Wrap(err, "while listing test definitions")
+
+	return s.findAll(ctx, suite)
+}
+
+func (s *Definition) findBySelector(ctx context.Context, suite v1alpha1.ClusterTestSuite) ([]v1alpha1.TestDefinition, error) {
+	byNames, err := s.findByNames(ctx, suite)
+	if err != nil {
+		return nil, err
 	}
-	return list.Items, nil
+
+	byLabelExpressions, err := s.findByLabelExpressions(ctx, suite)
+	if err != nil {
+		return nil, err
+	}
+
+	return s.unique(byNames, byLabelExpressions), nil
 }
 
 func (s *Definition) findByNames(ctx context.Context, suite v1alpha1.ClusterTestSuite) ([]v1alpha1.TestDefinition, error) {
-	var list []v1alpha1.TestDefinition
+	result := make([]v1alpha1.TestDefinition, 0)
 	for _, tRef := range suite.Spec.Selectors.MatchNames {
 		def := v1alpha1.TestDefinition{}
 		err := s.reader.Get(ctx, types.NamespacedName{Name: tRef.Name, Namespace: tRef.Namespace}, &def)
@@ -48,7 +60,47 @@ func (s *Definition) findByNames(ctx context.Context, suite v1alpha1.ClusterTest
 		default:
 			return nil, humanerr.NewError(wrappedErr, "Internal error")
 		}
-		list = append(list, def)
+		result = append(result, def)
+	}
+	return result, nil
+}
+
+func (s *Definition) findByLabelExpressions(ctx context.Context, suite v1alpha1.ClusterTestSuite) ([]v1alpha1.TestDefinition, error) {
+	result := make([]v1alpha1.TestDefinition, 0)
+	for _, expr := range suite.Spec.Selectors.MatchLabelExpressions {
+		selector, err := labels.Parse(expr)
+		if err != nil {
+			return nil, errors.Wrapf(err, "while parsing label expression [expression: %s]", expr)
+		}
+		var list v1alpha1.TestDefinitionList
+		if err := s.reader.List(ctx, &client.ListOptions{LabelSelector: selector}, &list); err != nil {
+			return nil, errors.Wrapf(err, "while fetching test definition from selector [expression: %s]", expr)
+		}
+		result = append(result, list.Items...)
+	}
+	return result, nil
+}
+
+func (s *Definition) unique(slices ...[]v1alpha1.TestDefinition) []v1alpha1.TestDefinition {
+	unique := make(map[types.UID]v1alpha1.TestDefinition)
+	for _, slice := range slices {
+		for _, td := range slice {
+			unique[td.UID] = td
+		}
+	}
+
+	result := make([]v1alpha1.TestDefinition, 0, len(unique))
+	for _, td := range unique {
+		result = append(result, td)
 	}
-	return list, nil
+
+	return result
+}
+
+func (s *Definition) findAll(ctx context.Context, suite v1alpha1.ClusterTestSuite) ([]v1alpha1.TestDefinition, error) {
+	var list v1alpha1.TestDefinitionList
+	if err := s.reader.List(ctx, &client.ListOptions{Namespace: ""}, &list); err != nil {
+		return nil, errors.Wrap(err, "while listing test definitions")
+	}
+	return list.Items, nil
 }