feat: Add global and stage job configuration lookup

docs/FEATURES.md

@@ -3,6 +3,7 @@
 
 - [Features](#features)
   - [Getting started](#getting-started)
+  - [Job Configuration](#job-configuration)
   - [Specifying the working directory](#specifying-the-working-directory)
   - [Event Matching](#event-matching)
   - [Kubernetes Job](#kubernetes-job)
@@ -35,6 +36,29 @@
 
 To get started with job-executor-service, please follow the [Quickstart](../README.md#quickstart).
 
+### Job Configuration
+
+The job-executor-service can be configured with the `job/config.yaml` configuration file, which should be uploaded as
+resource to Keptn. The job-executor-service will scan the resources in the following order to determine which configuration
+should be used: 
+- **Service**: A resource that is stored on the service level will always used by the job executor service if available 
+- **Stage**: If no service resource is found, the stage resources are searched for a job configuration
+- **Project**: If no other resources are round the job executor will fallback to a project wide configuration.
+
+If the job executor service can't find a configuration file, it will respond with an error event, which can be viewed
+in the uniform page of the Keptn bridge.
+
+A typical job configuration contains usually one or more actions that will be triggered when a specific event is
+received: 
+```yaml
+apiVersion: v2
+actions:
+  - name: "Print files"
+    events:
+      - name: "sh.keptn.event.sample.triggered"
+    tasks:
+      - ...
+```
 
 ### Specifying the working directory
 

pkg/config/reader.go

@@ -10,24 +10,52 @@ const jobConfigResourceName = "job/config.yaml"
 
 //go:generate mockgen -destination=fake/reader_mock.go -package=fake . KeptnResourceService
 
-// KeptnResourceService defines the contract used by JobConfigReader to retrieve a resource from keptn (using project,
-// service, stage from context)
+// KeptnResourceService defines the contract used by JobConfigReader to retrieve a resource from Keptn
+// The project, service, stage environment variables are taken from the context of the ResourceService (Event)
 type KeptnResourceService interface {
-	GetResource(resource string, gitCommitID string) ([]byte, error)
+	// GetServiceResource returns the service level resource
+	GetServiceResource(resource string, gitCommitID string) ([]byte, error)
+
+	// GetProjectResource returns the resource that was defined on project level
+	GetProjectResource(resource string, gitCommitID string) ([]byte, error)
+
+	// GetStageResource returns the resource that was defined in the stage
+	GetStageResource(resource string, gitCommitID string) ([]byte, error)
 }
 
 // JobConfigReader retrieves and parses job configuration from Keptn
 type JobConfigReader struct {
 	Keptn KeptnResourceService
 }
 
+// FindJobConfigResource searches for the job configuration resource in the service, stage and then the project
+// and returns the content of the first resource that is found
+func (jcr *JobConfigReader) FindJobConfigResource(gitCommitID string) ([]byte, error) {
+	if config, err := jcr.Keptn.GetServiceResource(jobConfigResourceName, gitCommitID); err == nil {
+		return config, nil
+	}
+
+	if config, err := jcr.Keptn.GetStageResource(jobConfigResourceName, gitCommitID); err == nil {
+		return config, nil
+	}
+
+	// FIXME: Since the resource service uses different branches, the commitID may not be in the main
+	//        branch and therefore it's not possible to query the project fallback configuration!
+	if config, err := jcr.Keptn.GetProjectResource(jobConfigResourceName, ""); err == nil {
+		return config, nil
+	}
+
+	// TODO: Improve error handling:
+	return nil, fmt.Errorf("unable to find job configuration")
+}
+
 // GetJobConfig retrieves job/config.yaml resource from keptn and parses it into a Config struct.
 // Additionally, also the SHA1 hash of the retrieved configuration will be returned.
 // In case of error retrieving the resource or parsing the yaml it will return (nil,
 // error) with the original error correctly wrapped in the local one
 func (jcr *JobConfigReader) GetJobConfig(gitCommitID string) (*Config, string, error) {
 
-	resource, err := jcr.Keptn.GetResource(jobConfigResourceName, gitCommitID)
+	resource, err := jcr.FindJobConfigResource(gitCommitID)
 	if err != nil {
 		return nil, "", fmt.Errorf("error retrieving job config: %w", err)
 	}

pkg/config/reader_test.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"errors"
+	"fmt"
 	"testing"
 
 	"github.com/golang/mock/gomock"
@@ -16,12 +17,16 @@ func TestConfigRetrievalFailed(t *testing.T) {
 
 	mockKeptnResourceService := fake.NewMockKeptnResourceService(mockCtrl)
 	retrievalError := errors.New("error getting resource")
-	mockKeptnResourceService.EXPECT().GetResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return(nil, retrievalError)
+	mockKeptnResourceService.EXPECT().GetServiceResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return(nil, retrievalError)
+	mockKeptnResourceService.EXPECT().GetStageResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return(nil, retrievalError)
+
+	// NOTE: fetching project resources works only without a git commit id, because of branches !
+	mockKeptnResourceService.EXPECT().GetProjectResource("job/config.yaml", "").Return(nil, retrievalError)
 
 	sut := JobConfigReader{Keptn: mockKeptnResourceService}
 
 	config, _, err := sut.GetJobConfig("c25692cb4fe4068fbdc2")
-	assert.ErrorIs(t, err, retrievalError)
+	assert.Error(t, err)
 	assert.Nil(t, config)
 }
 
@@ -35,7 +40,7 @@ func TestMalformedConfig(t *testing.T) {
                             has_nothing_to_do:
                                 with_job_executor: true
                     `
-	mockKeptnResourceService.EXPECT().GetResource("job/config.yaml", "").Return(
+	mockKeptnResourceService.EXPECT().GetServiceResource("job/config.yaml", "").Return(
 		[]byte(yamlConfig),
 		nil,
 	)
@@ -64,7 +69,7 @@ func TestGetConfigHappyPath(t *testing.T) {
                             cmd:
                                 - echo "Hello World!"
                     `
-	mockKeptnResourceService.EXPECT().GetResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return(
+	mockKeptnResourceService.EXPECT().GetServiceResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return(
 		[]byte(yamlConfig),
 		nil,
 	)
@@ -75,3 +80,96 @@ func TestGetConfigHappyPath(t *testing.T) {
 	assert.NoError(t, err)
 	assert.NotNil(t, config)
 }
+
+func TestJobConfigReader_FindJobConfigResource(t *testing.T) {
+
+	t.Run("Find in service", func(t *testing.T) {
+		mockCtrl := gomock.NewController(t)
+		mockKeptnResourceService := fake.NewMockKeptnResourceService(mockCtrl)
+
+		sut := JobConfigReader{Keptn: mockKeptnResourceService}
+
+		mockKeptnResourceService.EXPECT().GetServiceResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return(
+			[]byte("test"),
+			nil,
+		)
+
+		result, err := sut.FindJobConfigResource("c25692cb4fe4068fbdc2")
+		assert.NoError(t, err)
+		assert.Equal(t, result, []byte("test"))
+	})
+
+	t.Run("Find in stage", func(t *testing.T) {
+		mockCtrl := gomock.NewController(t)
+		mockKeptnResourceService := fake.NewMockKeptnResourceService(mockCtrl)
+
+		sut := JobConfigReader{Keptn: mockKeptnResourceService}
+
+		mockKeptnResourceService.EXPECT().GetServiceResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return(
+			nil,
+			fmt.Errorf("some error"),
+		)
+
+		mockKeptnResourceService.EXPECT().GetStageResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return(
+			[]byte("test1"),
+			nil,
+		)
+
+		result, err := sut.FindJobConfigResource("c25692cb4fe4068fbdc2")
+		assert.NoError(t, err)
+		assert.Equal(t, result, []byte("test1"))
+	})
+
+	t.Run("Find in project", func(t *testing.T) {
+		mockCtrl := gomock.NewController(t)
+		mockKeptnResourceService := fake.NewMockKeptnResourceService(mockCtrl)
+
+		sut := JobConfigReader{Keptn: mockKeptnResourceService}
+
+		mockKeptnResourceService.EXPECT().GetServiceResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return(
+			nil,
+			fmt.Errorf("some error"),
+		)
+
+		mockKeptnResourceService.EXPECT().GetStageResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return(
+			nil,
+			fmt.Errorf("some error"),
+		)
+
+		mockKeptnResourceService.EXPECT().GetProjectResource("job/config.yaml", "").Return(
+			[]byte("abc"),
+			nil,
+		)
+
+		result, err := sut.FindJobConfigResource("c25692cb4fe4068fbdc2")
+		assert.NoError(t, err)
+		assert.Equal(t, result, []byte("abc"))
+	})
+
+	t.Run("Not job config", func(t *testing.T) {
+		mockCtrl := gomock.NewController(t)
+		mockKeptnResourceService := fake.NewMockKeptnResourceService(mockCtrl)
+
+		sut := JobConfigReader{Keptn: mockKeptnResourceService}
+
+		mockKeptnResourceService.EXPECT().GetServiceResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return(
+			nil,
+			fmt.Errorf("some error"),
+		)
+
+		mockKeptnResourceService.EXPECT().GetStageResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return(
+			nil,
+			fmt.Errorf("some error"),
+		)
+
+		mockKeptnResourceService.EXPECT().GetProjectResource("job/config.yaml", "").Return(
+			nil,
+			fmt.Errorf("some error"),
+		)
+
+		result, err := sut.FindJobConfigResource("c25692cb4fe4068fbdc2")
+		assert.Error(t, err)
+		assert.Nil(t, result)
+	})
+
+}

pkg/file/file.go

@@ -2,7 +2,6 @@ package file
 
 import (
 	"fmt"
-	"keptn-contrib/job-executor-service/pkg/config"
 	"keptn-contrib/job-executor-service/pkg/keptn"
 	"log"
 	"path/filepath"
@@ -13,16 +12,11 @@ import (
 // MountFiles requests all specified files of a task from the keptn configuration service and copies them to /keptn
 func MountFiles(actionName string, taskName string, fs afero.Fs, configService keptn.ConfigService) error {
 
-	resource, err := configService.GetKeptnResource(fs, "job/config.yaml")
+	configuration, err := configService.GetJobConfiguration(fs)
 	if err != nil {
 		return fmt.Errorf("could not find config for job-executor-service: %v", err)
 	}
 
-	configuration, err := config.NewConfig(resource)
-	if err != nil {
-		return fmt.Errorf("could not parse config: %s", err)
-	}
-
 	found, action := configuration.FindActionByName(actionName)
 	if !found {
 		return fmt.Errorf("no action found with name '%s'", actionName)

pkg/file/file_test.go

@@ -2,6 +2,7 @@ package file
 
 import (
 	"errors"
+	config2 "keptn-contrib/job-executor-service/pkg/config"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -57,7 +58,8 @@ func TestMountFiles(t *testing.T) {
 	fs := afero.NewMemMapFs()
 	configServiceMock := CreateKeptnConfigServiceMock(t)
 
-	configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return([]byte(simpleConfig), nil)
+	config, _ := config2.NewConfig([]byte(simpleConfig))
+	configServiceMock.EXPECT().GetJobConfiguration(fs).Times(1).Return(config, nil)
 	configServiceMock.EXPECT().GetAllKeptnResources(
 		fs, "locust",
 	).Times(1).Return(
@@ -94,7 +96,7 @@ func TestMountFilesConfigFileNotFound(t *testing.T) {
 	fs := afero.NewMemMapFs()
 	configServiceMock := CreateKeptnConfigServiceMock(t)
 
-	configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return(nil, errors.New("not found"))
+	configServiceMock.EXPECT().GetJobConfiguration(fs).Times(1).Return(nil, errors.New("not found"))
 
 	err := MountFiles("action", "task", fs, configServiceMock)
 	require.Error(t, err)
@@ -106,7 +108,8 @@ func TestMountFilesConfigFileNotValid(t *testing.T) {
 	fs := afero.NewMemMapFs()
 	configServiceMock := CreateKeptnConfigServiceMock(t)
 
-	configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return([]byte(pythonFile), nil)
+	config, configErr := config2.NewConfig([]byte(pythonFile))
+	configServiceMock.EXPECT().GetJobConfiguration(fs).Times(1).Return(config, configErr)
 
 	err := MountFiles("action", "task", fs, configServiceMock)
 	require.Error(t, err)
@@ -118,7 +121,8 @@ func TestMountFilesNoActionMatch(t *testing.T) {
 	fs := afero.NewMemMapFs()
 	configServiceMock := CreateKeptnConfigServiceMock(t)
 
-	configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return([]byte(simpleConfig), nil)
+	config, _ := config2.NewConfig([]byte(simpleConfig))
+	configServiceMock.EXPECT().GetJobConfiguration(fs).Times(1).Return(config, nil)
 
 	err := MountFiles("actionNotMatching", "task", fs, configServiceMock)
 	require.Error(t, err)
@@ -130,7 +134,8 @@ func TestMountFilesNoTaskMatch(t *testing.T) {
 	fs := afero.NewMemMapFs()
 	configServiceMock := CreateKeptnConfigServiceMock(t)
 
-	configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return([]byte(simpleConfig), nil)
+	config, _ := config2.NewConfig([]byte(simpleConfig))
+	configServiceMock.EXPECT().GetJobConfiguration(fs).Times(1).Return(config, nil)
 
 	err := MountFiles("action", "taskNotMatching", fs, configServiceMock)
 	require.Error(t, err)
@@ -142,7 +147,8 @@ func TestMountFilesFileNotFound(t *testing.T) {
 	fs := afero.NewMemMapFs()
 	configServiceMock := CreateKeptnConfigServiceMock(t)
 
-	configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return([]byte(simpleConfig), nil)
+	config, _ := config2.NewConfig([]byte(simpleConfig))
+	configServiceMock.EXPECT().GetJobConfiguration(fs).Times(1).Return(config, nil)
 	configServiceMock.EXPECT().GetAllKeptnResources(fs, "/helm/values.yaml").Times(1).Return(
 		nil, errors.New("not found"),
 	)