diff --git a/config/crds/testing_v1alpha1_clustertestsuite.yaml b/config/crds/testing_v1alpha1_clustertestsuite.yaml index b287e19..8ae8d9f 100644 --- a/config/crds/testing_v1alpha1_clustertestsuite.yaml +++ b/config/crds/testing_v1alpha1_clustertestsuite.yaml @@ -53,9 +53,10 @@ spec: description: Decide which tests to execute. If not provided execute all tests properties: - matchLabels: - description: Find test definitions by it's labels. TestDefinition - should have AT LEAST one label listed here to be executed. + matchLabelExpressions: + description: 'Find test definitions by their labels. TestDefinition + must match AT LEAST one expression listed here to be executed. + For the complete grammar see: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels' items: type: string type: array diff --git a/config/samples/advanced/testsuite-select-by-labels.yaml b/config/samples/advanced/testsuite-select-by-labels.yaml new file mode 100644 index 0000000..f47a598 --- /dev/null +++ b/config/samples/advanced/testsuite-select-by-labels.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: testing.kyma-project.io/v1alpha1 +kind: ClusterTestSuite +metadata: + labels: + controller-tools.k8s.io: "1.0" + name: testsuite-selected-by-labels +spec: + count: 1 + selectors: + matchLabelExpressions: + # This example executes all not long tests for frontend and all tests for backend + - component=frontend,test-duration!=long + - component=backend + diff --git a/docs/crd-cluster-test-suite.md b/docs/crd-cluster-test-suite.md index e1b3f1a..95c56df 100644 --- a/docs/crd-cluster-test-suite.md +++ b/docs/crd-cluster-test-suite.md @@ -31,7 +31,7 @@ This table lists all the possible parameters of a given resource together with t | **metadata.name** | **YES** | Specifies the name of the CR. | | **spec.selectors** | **NO** | Defines which tests should be executed. You can define tests by specifying their names or labels. Selectors are additive. If not defined, all tests from all Namespaces are executed. | **spec.selectors.matchNames** | **NO** | Lists TestDefinitions to execute. For every element on the list, specify **name** and **namespace** that refers to a TestDefinition. | -| **spec.selectors.matchLabels** | **NO** | Lists labels that match labels of TestDefinitions to execute. A TestDefinition is selected if at least one label matches. This feature is not yet implemented. | +| **spec.selectors.matchLabelExpressions** | **NO** | Lists label expressions that match labels of TestDefinitions to execute. A TestDefinition is selected if at least one label expression matches. See [this](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels) document for more details. | | **spec.concurrency** | **NO** | Defines how many tests can be executed at the same time, which depends on cluster size and its load. The default value is `1`. | **spec.suiteTimeout** | **NO** | Defines the maximal suite duration after which test executions are interrupted and marked as **Failed**. The default value is one hour. This feature is not yet implemented. | **spec.count** | **NO** | Defines how many times every test should be executed. **Spec.Count** and **Spec.MaxRetries** are mutually exclusive. The default value is `1`. diff --git a/pkg/apis/testing/v1alpha1/testsuite_types.go b/pkg/apis/testing/v1alpha1/testsuite_types.go index 10bb3f0..3dee443 100644 --- a/pkg/apis/testing/v1alpha1/testsuite_types.go +++ b/pkg/apis/testing/v1alpha1/testsuite_types.go @@ -108,9 +108,10 @@ type TestSuiteSpec struct { type TestsSelector struct { // Find test definitions by it's name MatchNames []TestDefReference `json:"matchNames,omitempty"` - // Find test definitions by it's labels. - // TestDefinition should have AT LEAST one label listed here to be executed. - MatchLabels []string `json:"matchLabels,omitempty"` + // Find test definitions by their labels. + // TestDefinition must match AT LEAST one expression listed here to be executed. + // For the complete grammar see: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + MatchLabelExpressions []string `json:"matchLabelExpressions,omitempty"` } type TestDefReference struct { @@ -157,3 +158,7 @@ type TestExecution struct { func init() { SchemeBuilder.Register(&ClusterTestSuite{}, &ClusterTestSuiteList{}) } + +func (in ClusterTestSuite) HasSelector() bool { + return len(in.Spec.Selectors.MatchNames) > 0 || len(in.Spec.Selectors.MatchLabelExpressions) > 0 +} diff --git a/pkg/apis/testing/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/testing/v1alpha1/zz_generated.deepcopy.go index 4bd3d7e..a331b54 100644 --- a/pkg/apis/testing/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/testing/v1alpha1/zz_generated.deepcopy.go @@ -311,8 +311,8 @@ func (in *TestsSelector) DeepCopyInto(out *TestsSelector) { *out = make([]TestDefReference, len(*in)) copy(*out, *in) } - if in.MatchLabels != nil { - in, out := &in.MatchLabels, &out.MatchLabels + if in.MatchLabelExpressions != nil { + in, out := &in.MatchLabelExpressions, &out.MatchLabelExpressions *out = make([]string, len(*in)) copy(*out, *in) } diff --git a/pkg/controller/testsuite/testsuite_controller_test.go b/pkg/controller/testsuite/testsuite_controller_test.go index 33300e3..c544f68 100644 --- a/pkg/controller/testsuite/testsuite_controller_test.go +++ b/pkg/controller/testsuite/testsuite_controller_test.go @@ -330,6 +330,91 @@ func TestReconcileClusterTestSuite(t *testing.T) { }) + t.Run("selective testing", func(t *testing.T) { + // GIVEN + // Setup the Manager and Controller + mgr, err := manager.New(cfg, manager.Options{}) + require.NoError(t, err) + c := mgr.GetClient() + + testNs := generateTestNs() + ctx := context.Background() + + require.NoError(t, add(mgr, newReconciler(mgr))) + stopMgr, mgrStopped := StartTestManager(t, mgr) + + defer func() { + close(stopMgr) + mgrStopped.Wait() + }() + + logf.SetLogger(logf.ZapLogger(false)) + + podReconciler, err := startMockPodController(mgr, 0) + require.NoError(t, err) + + // WHEN + ns := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNs, + }, + } + err = c.Create(ctx, ns) + require.NoError(t, err) + defer cleanupK8sObject(ctx, c, ns) + + testA := getSequentialTest("test-a", testNs) + err = c.Create(ctx, testA) + require.NoError(t, err) + defer cleanupK8sObject(ctx, c, testA) + + testB := getSequentialTest("test-b", testNs) + testB.Labels = map[string]string{"test": "true"} + err = c.Create(ctx, testB) + require.NoError(t, err) + defer cleanupK8sObject(ctx, c, testB) + + testC := getSequentialTest("test-c", testNs) + testC.Labels = map[string]string{"test": "false"} + err = c.Create(ctx, testC) + require.NoError(t, err) + defer cleanupK8sObject(ctx, c, testC) + + suite := &testingv1alpha1.ClusterTestSuite{ + ObjectMeta: metav1.ObjectMeta{Name: "suite-selective"}, + Spec: testingv1alpha1.TestSuiteSpec{ + Concurrency: 1, + Count: 1, + Selectors: testingv1alpha1.TestsSelector{ + MatchNames: []testingv1alpha1.TestDefReference{ + { + Name: "test-a", + Namespace: testNs, + }, + }, + MatchLabelExpressions: []string{ + "test=true", + }, + }, + }, + } + err = c.Create(ctx, suite) + require.NoError(t, err) + defer cleanupK8sObject(ctx, c, suite) + + // THEN + repeat.FuncAtMost(t, func() error { + return checkIfsuiteIsSucceeded(ctx, c, "suite-selective") + }, defaultAssertionTimeout) + + repeat.FuncAtMost(t, func() error { + return checkIfPodsWereCreated(ctx, c, testNs, []string{ + "oct-tp-suite-selective-test-a-0", + "oct-tp-suite-selective-test-b-0"}) + }, defaultAssertionTimeout) + + assertThatPodsCreatedSequentially(t, podReconciler.getAppliedChanges()) + }) } func assertThatPodsCreatedConcurrently(t *testing.T, appliedChanges []podStatusChanges) { @@ -418,7 +503,7 @@ type mockPodReconciler struct { cli client.Client // mutex protecting access to applied changes slice mtx sync.Mutex - // chronologically list of pod changes. Use getter to access it autside the struct + // chronologically list of pod changes. Use getter to access it outside the struct appliedChanges []podStatusChanges enforceNumberOfConcurrentlyRunningPods int reconcileAfter time.Duration @@ -483,8 +568,8 @@ func (r *mockPodReconciler) getLogger() logr.Logger { func startMockPodController(mgr manager.Manager, enforceConcurrentlyRunningPods int) (*mockPodReconciler, error) { pr := &mockPodReconciler{ - cli: mgr.GetClient(), - mtx: sync.Mutex{}, + cli: mgr.GetClient(), + mtx: sync.Mutex{}, enforceNumberOfConcurrentlyRunningPods: enforceConcurrentlyRunningPods, reconcileAfter: time.Millisecond * 100, } diff --git a/pkg/fetcher/definition.go b/pkg/fetcher/definition.go index 346d070..734245c 100644 --- a/pkg/fetcher/definition.go +++ b/pkg/fetcher/definition.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/kyma-incubator/octopus/pkg/humanerr" k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "github.com/kyma-incubator/octopus/pkg/apis/testing/v1alpha1" @@ -24,19 +25,30 @@ type Definition struct { func (s *Definition) FindMatching(suite v1alpha1.ClusterTestSuite) ([]v1alpha1.TestDefinition, error) { ctx := context.TODO() - if len(suite.Spec.Selectors.MatchNames) > 0 { - return s.findByNames(ctx, suite) + + if suite.HasSelector() { + return s.findBySelector(ctx, suite) } - // TODO(aszecowka) later so far we return all test definitions for all namespaces (https://github.com/kyma-incubator/octopus/issues/7) - var list v1alpha1.TestDefinitionList - if err := s.reader.List(ctx, &client.ListOptions{Namespace: ""}, &list); err != nil { - return nil, errors.Wrap(err, "while listing test definitions") + + return s.findAll(ctx, suite) +} + +func (s *Definition) findBySelector(ctx context.Context, suite v1alpha1.ClusterTestSuite) ([]v1alpha1.TestDefinition, error) { + byNames, err := s.findByNames(ctx, suite) + if err != nil { + return nil, err } - return list.Items, nil + + byLabelExpressions, err := s.findByLabelExpressions(ctx, suite) + if err != nil { + return nil, err + } + + return s.unique(byNames, byLabelExpressions), nil } func (s *Definition) findByNames(ctx context.Context, suite v1alpha1.ClusterTestSuite) ([]v1alpha1.TestDefinition, error) { - var list []v1alpha1.TestDefinition + result := make([]v1alpha1.TestDefinition, 0) for _, tRef := range suite.Spec.Selectors.MatchNames { def := v1alpha1.TestDefinition{} err := s.reader.Get(ctx, types.NamespacedName{Name: tRef.Name, Namespace: tRef.Namespace}, &def) @@ -48,7 +60,47 @@ func (s *Definition) findByNames(ctx context.Context, suite v1alpha1.ClusterTest default: return nil, humanerr.NewError(wrappedErr, "Internal error") } - list = append(list, def) + result = append(result, def) + } + return result, nil +} + +func (s *Definition) findByLabelExpressions(ctx context.Context, suite v1alpha1.ClusterTestSuite) ([]v1alpha1.TestDefinition, error) { + result := make([]v1alpha1.TestDefinition, 0) + for _, expr := range suite.Spec.Selectors.MatchLabelExpressions { + selector, err := labels.Parse(expr) + if err != nil { + return nil, errors.Wrapf(err, "while parsing label expression [expression: %s]", expr) + } + var list v1alpha1.TestDefinitionList + if err := s.reader.List(ctx, &client.ListOptions{LabelSelector: selector}, &list); err != nil { + return nil, errors.Wrapf(err, "while fetching test definition from selector [expression: %s]", expr) + } + result = append(result, list.Items...) + } + return result, nil +} + +func (s *Definition) unique(slices ...[]v1alpha1.TestDefinition) []v1alpha1.TestDefinition { + unique := make(map[types.UID]v1alpha1.TestDefinition) + for _, slice := range slices { + for _, td := range slice { + unique[td.UID] = td + } + } + + result := make([]v1alpha1.TestDefinition, 0, len(unique)) + for _, td := range unique { + result = append(result, td) } - return list, nil + + return result +} + +func (s *Definition) findAll(ctx context.Context, suite v1alpha1.ClusterTestSuite) ([]v1alpha1.TestDefinition, error) { + var list v1alpha1.TestDefinitionList + if err := s.reader.List(ctx, &client.ListOptions{Namespace: ""}, &list); err != nil { + return nil, errors.Wrap(err, "while listing test definitions") + } + return list.Items, nil } diff --git a/pkg/fetcher/definition_test.go b/pkg/fetcher/definition_test.go index 122b083..0ab138c 100644 --- a/pkg/fetcher/definition_test.go +++ b/pkg/fetcher/definition_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "github.com/kyma-incubator/octopus/pkg/humanerr" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "testing" @@ -41,18 +42,21 @@ func TestFindMatching(t *testing.T) { // GIVEN testA := &v1alpha1.TestDefinition{ ObjectMeta: v1.ObjectMeta{ + UID: "test-uid-a", Name: "test-a", Namespace: "test-a", }, } testB := &v1alpha1.TestDefinition{ ObjectMeta: v1.ObjectMeta{ + UID: "test-uid-b", Name: "test-b", Namespace: "test-b", }, } testC := &v1alpha1.TestDefinition{ ObjectMeta: v1.ObjectMeta{ + UID: "test-uid-c", Name: "test-c", Namespace: "test-c", }, @@ -87,6 +91,103 @@ func TestFindMatching(t *testing.T) { }) + t.Run("return tests selected by label expressions", func(t *testing.T) { + // GIVEN + testA := &v1alpha1.TestDefinition{ + ObjectMeta: v1.ObjectMeta{ + UID: "test-uid-a", + Name: "test-a", + Namespace: "test-a", + Labels: map[string]string{ + "test": "true", + }, + }, + } + testB := &v1alpha1.TestDefinition{ + ObjectMeta: v1.ObjectMeta{ + UID: "test-uid-b", + Name: "test-b", + Namespace: "test-b", + Labels: map[string]string{ + "test": "false", + }, + }, + } + testC := &v1alpha1.TestDefinition{ + ObjectMeta: v1.ObjectMeta{ + UID: "test-uid-c", + Name: "test-c", + Namespace: "test-c", + Labels: map[string]string{ + "other": "123", + }, + }, + } + + fakeCli := fake.NewFakeClientWithScheme(sch, + testA, testB, testC, + ) + mockReader := &mockListReader{fakeCli: fakeCli} + + service := fetcher.NewForDefinition(mockReader) + // WHEN + out, err := service.FindMatching(v1alpha1.ClusterTestSuite{ + Spec: v1alpha1.TestSuiteSpec{ + Selectors: v1alpha1.TestsSelector{ + MatchLabelExpressions: []string{ + "other", + "test=true", + }, + }, + }, + }) + // THEN + require.NoError(t, err) + assert.Len(t, out, 2) + assert.Contains(t, out, *testA) + assert.Contains(t, out, *testC) + }) + + t.Run("return tests returns unique result across all selectors", func(t *testing.T) { + // GIVEN + testA := &v1alpha1.TestDefinition{ + ObjectMeta: v1.ObjectMeta{ + UID: "test-uid-a", + Name: "test-a", + Namespace: "test-a", + Labels: map[string]string{ + "test": "true", + }, + }, + } + + fakeCli := fake.NewFakeClientWithScheme(sch, + testA, + ) + mockReader := &mockListReader{fakeCli: fakeCli} + service := fetcher.NewForDefinition(mockReader) + // WHEN + out, err := service.FindMatching(v1alpha1.ClusterTestSuite{ + Spec: v1alpha1.TestSuiteSpec{ + Selectors: v1alpha1.TestsSelector{ + MatchNames: []v1alpha1.TestDefReference{ + { + Name: "test-a", + Namespace: "test-a", + }, + }, + MatchLabelExpressions: []string{ + "test=true", + }, + }, + }, + }) + // THEN + require.NoError(t, err) + assert.Len(t, out, 1) + assert.Contains(t, out, *testA) + }) + t.Run("return error if test selected by name does not exist", func(t *testing.T) { // GIVEN fakeCli := fake.NewFakeClientWithScheme(sch) @@ -113,7 +214,7 @@ func TestFindMatching(t *testing.T) { t.Run("return internal error when fetching selected tests failed", func(t *testing.T) { // GIVEN - errClient := &mockErrReader{err:errors.New("some error")} + errClient := &mockErrReader{err: errors.New("some error")} service := fetcher.NewForDefinition(errClient) // WHEN @@ -150,3 +251,33 @@ func (m *mockErrReader) Get(ctx context.Context, key client.ObjectKey, obj runti func (m *mockErrReader) List(ctx context.Context, opts *client.ListOptions, list runtime.Object) error { return m.err } + +type mockListReader struct { + fakeCli client.Reader +} + +func (m *mockListReader) Get(ctx context.Context, key client.ObjectKey, obj runtime.Object) error { + return m.fakeCli.Get(ctx, key, obj) +} + +func (m *mockListReader) List(ctx context.Context, opts *client.ListOptions, list runtime.Object) error { + // fakeCli has a bug fixed in controller-runtime 0.1.11 and it does not filter by labels. This mock can be removed + // when we update to new controller-runtime + // See: https://github.com/kubernetes-sigs/controller-runtime/issues/293 + if opts.LabelSelector == nil { + return m.fakeCli.List(ctx, opts, list) + } + + result := v1alpha1.TestDefinitionList{} + err := m.fakeCli.List(ctx, opts, &result) + if err != nil { + return err + } + + for _, td := range result.Items { + if opts.LabelSelector.Matches(labels.Set(td.Labels)) { + list.(*v1alpha1.TestDefinitionList).Items = append(list.(*v1alpha1.TestDefinitionList).Items, td) + } + } + return nil +}