diff --git a/docs/FEATURES.md b/docs/FEATURES.md index 6db15263..ebdd4f08 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -3,6 +3,7 @@ - [Features](#features) - [Getting started](#getting-started) + - [Job Configuration](#job-configuration) - [Specifying the working directory](#specifying-the-working-directory) - [Event Matching](#event-matching) - [Kubernetes Job](#kubernetes-job) @@ -35,6 +36,30 @@ To get started with job-executor-service, please follow the [Quickstart](../README.md#quickstart). +### Job Configuration + +The job-executor-service can be configured with the `job/config.yaml` configuration file, which should be uploaded as +resource to Keptn. The job-executor-service will scan the resources in the following order to determine which configuration +should be used: +- **Service**: A resource that is stored on the service level will always used by the job executor service if available +- **Stage**: If no service resource is found, the stage resources are searched for a job configuration +- **Project**: If no other resources are found the job executor will fallback to a project wide configuration + *(Note: the latest version of the project wide configuration file will be fetched!)* + +If the job executor service can't find a configuration file, it will respond with an error event, which can be viewed +in the uniform page of the Keptn bridge. + +A typical job configuration usually contains one or more actions that will be triggered when a specific event is +received: +```yaml +apiVersion: v2 +actions: + - name: "Print files" + events: + - name: "sh.keptn.event.sample.triggered" + tasks: + - ... +``` ### Specifying the working directory diff --git a/pkg/config/fake/reader_mock.go b/pkg/config/fake/reader_mock.go index 51d0509c..a6ab2310 100644 --- a/pkg/config/fake/reader_mock.go +++ b/pkg/config/fake/reader_mock.go @@ -33,17 +33,47 @@ func (m *MockKeptnResourceService) EXPECT() *MockKeptnResourceServiceMockRecorde return m.recorder } -// GetResource mocks base method. -func (m *MockKeptnResourceService) GetResource(arg0, arg1 string) ([]byte, error) { +// GetProjectResource mocks base method. +func (m *MockKeptnResourceService) GetProjectResource(arg0, arg1 string) ([]byte, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetResource", arg0, arg1) + ret := m.ctrl.Call(m, "GetProjectResource", arg0, arg1) ret0, _ := ret[0].([]byte) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetResource indicates an expected call of GetResource. -func (mr *MockKeptnResourceServiceMockRecorder) GetResource(arg0, arg1 interface{}) *gomock.Call { +// GetProjectResource indicates an expected call of GetProjectResource. +func (mr *MockKeptnResourceServiceMockRecorder) GetProjectResource(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResource", reflect.TypeOf((*MockKeptnResourceService)(nil).GetResource), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProjectResource", reflect.TypeOf((*MockKeptnResourceService)(nil).GetProjectResource), arg0, arg1) +} + +// GetServiceResource mocks base method. +func (m *MockKeptnResourceService) GetServiceResource(arg0, arg1 string) ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetServiceResource", arg0, arg1) + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetServiceResource indicates an expected call of GetServiceResource. +func (mr *MockKeptnResourceServiceMockRecorder) GetServiceResource(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetServiceResource", reflect.TypeOf((*MockKeptnResourceService)(nil).GetServiceResource), arg0, arg1) +} + +// GetStageResource mocks base method. +func (m *MockKeptnResourceService) GetStageResource(arg0, arg1 string) ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetStageResource", arg0, arg1) + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetStageResource indicates an expected call of GetStageResource. +func (mr *MockKeptnResourceServiceMockRecorder) GetStageResource(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStageResource", reflect.TypeOf((*MockKeptnResourceService)(nil).GetStageResource), arg0, arg1) } diff --git a/pkg/config/reader.go b/pkg/config/reader.go index 9f0a2a30..a9d49c4e 100644 --- a/pkg/config/reader.go +++ b/pkg/config/reader.go @@ -10,10 +10,17 @@ const jobConfigResourceName = "job/config.yaml" //go:generate mockgen -destination=fake/reader_mock.go -package=fake . KeptnResourceService -// KeptnResourceService defines the contract used by JobConfigReader to retrieve a resource from keptn (using project, -// service, stage from context) +// KeptnResourceService defines the contract used by JobConfigReader to retrieve a resource from Keptn +// The project, service, stage environment variables are taken from the context of the ResourceService (Event) type KeptnResourceService interface { - GetResource(resource string, gitCommitID string) ([]byte, error) + // GetServiceResource returns the service level resource + GetServiceResource(resource string, gitCommitID string) ([]byte, error) + + // GetProjectResource returns the resource that was defined on project level + GetProjectResource(resource string, gitCommitID string) ([]byte, error) + + // GetStageResource returns the resource that was defined in the stage + GetStageResource(resource string, gitCommitID string) ([]byte, error) } // JobConfigReader retrieves and parses job configuration from Keptn @@ -21,13 +28,33 @@ type JobConfigReader struct { Keptn KeptnResourceService } +// FindJobConfigResource searches for the job configuration resource in the service, stage and then the project +// and returns the content of the first resource that is found +func (jcr *JobConfigReader) FindJobConfigResource(gitCommitID string) ([]byte, error) { + if config, err := jcr.Keptn.GetServiceResource(jobConfigResourceName, gitCommitID); err == nil { + return config, nil + } + + if config, err := jcr.Keptn.GetStageResource(jobConfigResourceName, gitCommitID); err == nil { + return config, nil + } + + // NOTE: Since the resource service uses different branches, the commitID may not be in the main + // branch and therefore it's not possible to query the project fallback configuration! + if config, err := jcr.Keptn.GetProjectResource(jobConfigResourceName, ""); err == nil { + return config, nil + } + + return nil, fmt.Errorf("unable to find job configuration") +} + // GetJobConfig retrieves job/config.yaml resource from keptn and parses it into a Config struct. // Additionally, also the SHA1 hash of the retrieved configuration will be returned. // In case of error retrieving the resource or parsing the yaml it will return (nil, // error) with the original error correctly wrapped in the local one func (jcr *JobConfigReader) GetJobConfig(gitCommitID string) (*Config, string, error) { - resource, err := jcr.Keptn.GetResource(jobConfigResourceName, gitCommitID) + resource, err := jcr.FindJobConfigResource(gitCommitID) if err != nil { return nil, "", fmt.Errorf("error retrieving job config: %w", err) } diff --git a/pkg/config/reader_test.go b/pkg/config/reader_test.go index 8484b69b..0a332e60 100644 --- a/pkg/config/reader_test.go +++ b/pkg/config/reader_test.go @@ -2,6 +2,7 @@ package config import ( "errors" + "fmt" "testing" "github.com/golang/mock/gomock" @@ -16,12 +17,16 @@ func TestConfigRetrievalFailed(t *testing.T) { mockKeptnResourceService := fake.NewMockKeptnResourceService(mockCtrl) retrievalError := errors.New("error getting resource") - mockKeptnResourceService.EXPECT().GetResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return(nil, retrievalError) + mockKeptnResourceService.EXPECT().GetServiceResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return(nil, retrievalError) + mockKeptnResourceService.EXPECT().GetStageResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return(nil, retrievalError) + + // NOTE: fetching project resources works only without a git commit id, because of branches ! + mockKeptnResourceService.EXPECT().GetProjectResource("job/config.yaml", "").Return(nil, retrievalError) sut := JobConfigReader{Keptn: mockKeptnResourceService} config, _, err := sut.GetJobConfig("c25692cb4fe4068fbdc2") - assert.ErrorIs(t, err, retrievalError) + assert.Error(t, err) assert.Nil(t, config) } @@ -35,7 +40,7 @@ func TestMalformedConfig(t *testing.T) { has_nothing_to_do: with_job_executor: true ` - mockKeptnResourceService.EXPECT().GetResource("job/config.yaml", "").Return( + mockKeptnResourceService.EXPECT().GetServiceResource("job/config.yaml", "").Return( []byte(yamlConfig), nil, ) @@ -64,7 +69,7 @@ func TestGetConfigHappyPath(t *testing.T) { cmd: - echo "Hello World!" ` - mockKeptnResourceService.EXPECT().GetResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return( + mockKeptnResourceService.EXPECT().GetServiceResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return( []byte(yamlConfig), nil, ) @@ -75,3 +80,96 @@ func TestGetConfigHappyPath(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, config) } + +func TestJobConfigReader_FindJobConfigResource(t *testing.T) { + + t.Run("Find in service", func(t *testing.T) { + mockCtrl := gomock.NewController(t) + mockKeptnResourceService := fake.NewMockKeptnResourceService(mockCtrl) + + sut := JobConfigReader{Keptn: mockKeptnResourceService} + + mockKeptnResourceService.EXPECT().GetServiceResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return( + []byte("test"), + nil, + ) + + result, err := sut.FindJobConfigResource("c25692cb4fe4068fbdc2") + assert.NoError(t, err) + assert.Equal(t, result, []byte("test")) + }) + + t.Run("Find in stage", func(t *testing.T) { + mockCtrl := gomock.NewController(t) + mockKeptnResourceService := fake.NewMockKeptnResourceService(mockCtrl) + + sut := JobConfigReader{Keptn: mockKeptnResourceService} + + mockKeptnResourceService.EXPECT().GetServiceResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return( + nil, + fmt.Errorf("some error"), + ) + + mockKeptnResourceService.EXPECT().GetStageResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return( + []byte("test1"), + nil, + ) + + result, err := sut.FindJobConfigResource("c25692cb4fe4068fbdc2") + assert.NoError(t, err) + assert.Equal(t, result, []byte("test1")) + }) + + t.Run("Find in project", func(t *testing.T) { + mockCtrl := gomock.NewController(t) + mockKeptnResourceService := fake.NewMockKeptnResourceService(mockCtrl) + + sut := JobConfigReader{Keptn: mockKeptnResourceService} + + mockKeptnResourceService.EXPECT().GetServiceResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return( + nil, + fmt.Errorf("some error"), + ) + + mockKeptnResourceService.EXPECT().GetStageResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return( + nil, + fmt.Errorf("some error"), + ) + + mockKeptnResourceService.EXPECT().GetProjectResource("job/config.yaml", "").Return( + []byte("abc"), + nil, + ) + + result, err := sut.FindJobConfigResource("c25692cb4fe4068fbdc2") + assert.NoError(t, err) + assert.Equal(t, result, []byte("abc")) + }) + + t.Run("Not job config", func(t *testing.T) { + mockCtrl := gomock.NewController(t) + mockKeptnResourceService := fake.NewMockKeptnResourceService(mockCtrl) + + sut := JobConfigReader{Keptn: mockKeptnResourceService} + + mockKeptnResourceService.EXPECT().GetServiceResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return( + nil, + fmt.Errorf("some error"), + ) + + mockKeptnResourceService.EXPECT().GetStageResource("job/config.yaml", "c25692cb4fe4068fbdc2").Return( + nil, + fmt.Errorf("some error"), + ) + + mockKeptnResourceService.EXPECT().GetProjectResource("job/config.yaml", "").Return( + nil, + fmt.Errorf("some error"), + ) + + result, err := sut.FindJobConfigResource("c25692cb4fe4068fbdc2") + assert.Error(t, err) + assert.Nil(t, result) + }) + +} diff --git a/pkg/file/file.go b/pkg/file/file.go index 8155610c..88484695 100644 --- a/pkg/file/file.go +++ b/pkg/file/file.go @@ -2,7 +2,6 @@ package file import ( "fmt" - "keptn-contrib/job-executor-service/pkg/config" "keptn-contrib/job-executor-service/pkg/keptn" "log" "path/filepath" @@ -13,16 +12,11 @@ import ( // MountFiles requests all specified files of a task from the keptn configuration service and copies them to /keptn func MountFiles(actionName string, taskName string, fs afero.Fs, configService keptn.ConfigService) error { - resource, err := configService.GetKeptnResource(fs, "job/config.yaml") + configuration, err := configService.GetJobConfiguration() if err != nil { return fmt.Errorf("could not find config for job-executor-service: %v", err) } - configuration, err := config.NewConfig(resource) - if err != nil { - return fmt.Errorf("could not parse config: %s", err) - } - found, action := configuration.FindActionByName(actionName) if !found { return fmt.Errorf("no action found with name '%s'", actionName) diff --git a/pkg/file/file_test.go b/pkg/file/file_test.go index f56acb8f..fab4222e 100644 --- a/pkg/file/file_test.go +++ b/pkg/file/file_test.go @@ -2,12 +2,12 @@ package file import ( "errors" + config2 "keptn-contrib/job-executor-service/pkg/config" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "keptn-contrib/job-executor-service/pkg/keptn" keptnfake "keptn-contrib/job-executor-service/pkg/keptn/fake" "github.com/golang/mock/gomock" @@ -57,7 +57,8 @@ func TestMountFiles(t *testing.T) { fs := afero.NewMemMapFs() configServiceMock := CreateKeptnConfigServiceMock(t) - configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return([]byte(simpleConfig), nil) + config, _ := config2.NewConfig([]byte(simpleConfig)) + configServiceMock.EXPECT().GetJobConfiguration().Times(1).Return(config, nil) configServiceMock.EXPECT().GetAllKeptnResources( fs, "locust", ).Times(1).Return( @@ -94,7 +95,7 @@ func TestMountFilesConfigFileNotFound(t *testing.T) { fs := afero.NewMemMapFs() configServiceMock := CreateKeptnConfigServiceMock(t) - configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return(nil, errors.New("not found")) + configServiceMock.EXPECT().GetJobConfiguration().Times(1).Return(nil, errors.New("not found")) err := MountFiles("action", "task", fs, configServiceMock) require.Error(t, err) @@ -106,7 +107,8 @@ func TestMountFilesConfigFileNotValid(t *testing.T) { fs := afero.NewMemMapFs() configServiceMock := CreateKeptnConfigServiceMock(t) - configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return([]byte(pythonFile), nil) + config, configErr := config2.NewConfig([]byte(pythonFile)) + configServiceMock.EXPECT().GetJobConfiguration().Times(1).Return(config, configErr) err := MountFiles("action", "task", fs, configServiceMock) require.Error(t, err) @@ -118,7 +120,8 @@ func TestMountFilesNoActionMatch(t *testing.T) { fs := afero.NewMemMapFs() configServiceMock := CreateKeptnConfigServiceMock(t) - configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return([]byte(simpleConfig), nil) + config, _ := config2.NewConfig([]byte(simpleConfig)) + configServiceMock.EXPECT().GetJobConfiguration().Times(1).Return(config, nil) err := MountFiles("actionNotMatching", "task", fs, configServiceMock) require.Error(t, err) @@ -130,7 +133,8 @@ func TestMountFilesNoTaskMatch(t *testing.T) { fs := afero.NewMemMapFs() configServiceMock := CreateKeptnConfigServiceMock(t) - configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return([]byte(simpleConfig), nil) + config, _ := config2.NewConfig([]byte(simpleConfig)) + configServiceMock.EXPECT().GetJobConfiguration().Times(1).Return(config, nil) err := MountFiles("action", "taskNotMatching", fs, configServiceMock) require.Error(t, err) @@ -142,7 +146,8 @@ func TestMountFilesFileNotFound(t *testing.T) { fs := afero.NewMemMapFs() configServiceMock := CreateKeptnConfigServiceMock(t) - configServiceMock.EXPECT().GetKeptnResource(fs, "job/config.yaml").Times(1).Return([]byte(simpleConfig), nil) + config, _ := config2.NewConfig([]byte(simpleConfig)) + configServiceMock.EXPECT().GetJobConfiguration().Times(1).Return(config, nil) configServiceMock.EXPECT().GetAllKeptnResources(fs, "/helm/values.yaml").Times(1).Return( nil, errors.New("not found"), ) @@ -151,27 +156,3 @@ func TestMountFilesFileNotFound(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "not found") } - -func TestMountFilesWithLocalFileSystem(t *testing.T) { - - fs := afero.NewMemMapFs() - configService := keptn.NewConfigService(true, keptn.EventProperties{}, nil) - err := afero.WriteFile(fs, "job/config.yaml", []byte(simpleConfig), 0644) - assert.NoError(t, err) - err = afero.WriteFile(fs, "/helm/values.yaml", []byte("here be awesome configuration"), 0644) - assert.NoError(t, err) - err = afero.WriteFile(fs, "locust/basic.py", []byte("here be awesome test code"), 0644) - assert.NoError(t, err) - err = afero.WriteFile(fs, "locust/functional.py", []byte("here be more awesome test code"), 0644) - assert.NoError(t, err) - - err = MountFiles("action", "task", fs, configService) - assert.NoError(t, err) - - _, err = fs.Stat("/keptn/helm/values.yaml") - assert.NoError(t, err) - _, err = fs.Stat("/keptn/locust/basic.py") - assert.NoError(t, err) - _, err = fs.Stat("/keptn/locust/functional.py") - assert.NoError(t, err) -} diff --git a/pkg/keptn/config_service.go b/pkg/keptn/config_service.go index 3f12d9c0..6704ff48 100644 --- a/pkg/keptn/config_service.go +++ b/pkg/keptn/config_service.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/keptn/go-utils/pkg/api/models" api "github.com/keptn/go-utils/pkg/api/utils/v2" + "keptn-contrib/job-executor-service/pkg/config" "net/url" "os" "strings" @@ -14,10 +15,13 @@ import ( //go:generate mockgen -source=config_service.go -destination=fake/config_service_mock.go -package=fake ConfigService -// ConfigService provides methods to retrieve and match resources from the keptn configuration service +// ConfigService provides methods to retrieve and match resources from the keptn resource service type ConfigService interface { GetKeptnResource(fs afero.Fs, resource string) ([]byte, error) GetAllKeptnResources(fs afero.Fs, resource string) (map[string][]byte, error) + + // GetJobConfiguration returns the fetched job configuration + GetJobConfiguration() (*config.Config, error) } //go:generate mockgen -source=config_service.go -destination=fake/config_service_mock.go -package=fake V2ResourceHandler @@ -36,6 +40,7 @@ type configServiceImpl struct { useLocalFileSystem bool eventProperties EventProperties resourceHandler V2ResourceHandler + jobConfigReader config.JobConfigReader } // EventProperties represents a set of properties of a given cloud event @@ -48,40 +53,44 @@ type EventProperties struct { // NewConfigService creates and returns new ConfigService func NewConfigService(useLocalFileSystem bool, event EventProperties, resourceHandler V2ResourceHandler) ConfigService { - return &configServiceImpl{ - useLocalFileSystem: useLocalFileSystem, - eventProperties: event, - resourceHandler: resourceHandler, - } -} - -// GetKeptnResource returns a resource from the configuration repo based on the incoming cloud events project, service and stage -func (k *configServiceImpl) GetKeptnResource(fs afero.Fs, resource string) ([]byte, error) { + configurationService := new(configServiceImpl) - // if we run in a runlocal mode we are just getting the file from the local disk - if k.useLocalFileSystem { - return k.getKeptnResourceFromLocal(fs, resource) + configurationService.useLocalFileSystem = useLocalFileSystem + configurationService.eventProperties = event + configurationService.resourceHandler = resourceHandler + configurationService.jobConfigReader = config.JobConfigReader{ + Keptn: configurationService, } - scope := api.NewResourceScope() - scope.Project(k.eventProperties.Project) - scope.Stage(k.eventProperties.Stage) - scope.Service(k.eventProperties.Service) - - // NOTE: No idea why, but the API requires a double query escape for a path element and does not accept leading / - // while emitting absolute paths in the response ... - scope.Resource(url.QueryEscape(strings.TrimPrefix(resource, "/"))) + return configurationService +} +// buildResourceHandlerV2Options builds the URIOption list such that it contains a well formatted gitCommitID +func buildResourceHandlerV2Options(gitCommitID string) api.ResourcesGetResourceOptions { options := api.ResourcesGetResourceOptions{} - if k.eventProperties.GitCommitID != "" { + + if gitCommitID != "" { options.URIOptions = []api.URIOption{ api.AppendQuery(url.Values{ - "gitCommitID": []string{k.eventProperties.GitCommitID}, + "gitCommitID": []string{gitCommitID}, }), } } - requestedResource, err := k.resourceHandler.GetResource(context.Background(), *scope, options) + return options +} + +// fetchKeptnResource sets the resource in the scope correctly and fetches the resource from Keptn with the given gitCommitID +func (k *configServiceImpl) fetchKeptnResource(resource string, scope *api.ResourceScope, gitCommitID string) ([]byte, error) { + // NOTE: No idea why, but the API requires a double query escape for a path element and does not accept leading / + // while emitting absolute paths in the response ... + scope.Resource(url.QueryEscape(strings.TrimPrefix(resource, "/"))) + + requestedResource, err := k.resourceHandler.GetResource( + context.Background(), + *scope, + buildResourceHandlerV2Options(gitCommitID), + ) // return Nil in case resource couldn't be retrieved if err != nil || requestedResource.ResourceContent == "" { @@ -91,6 +100,23 @@ func (k *configServiceImpl) GetKeptnResource(fs afero.Fs, resource string) ([]by return []byte(requestedResource.ResourceContent), nil } +// GetKeptnResource returns a resource from the configuration repo based on the incoming cloud events project, service and stage +func (k *configServiceImpl) GetKeptnResource(fs afero.Fs, resource string) ([]byte, error) { + + // if we run in a runlocal mode we are just getting the file from the local disk + if k.useLocalFileSystem { + return k.getKeptnResourceFromLocal(fs, resource) + } + + scope := api.NewResourceScope() + scope.Project(k.eventProperties.Project) + scope.Stage(k.eventProperties.Stage) + scope.Service(k.eventProperties.Service) + + // finally download the resource: + return k.fetchKeptnResource(resource, scope, k.eventProperties.GitCommitID) +} + // GetAllKeptnResources returns a map of keptn resources (key=URI, value=content) from the configuration repo with // prefix 'resource' (matched with and without leading '/') func (k *configServiceImpl) GetAllKeptnResources(fs afero.Fs, resource string) (map[string][]byte, error) { @@ -112,7 +138,6 @@ func (k *configServiceImpl) GetAllKeptnResources(fs afero.Fs, resource string) ( // NOTE: // Since no exact file has been found, we have to assume that the given resource is a directory. // Directories don't really exist in the API, so we have to use a HasPrefix match here - scope := api.NewResourceScope() scope.Project(k.eventProperties.Project) scope.Stage(k.eventProperties.Stage) @@ -149,6 +174,32 @@ func (k *configServiceImpl) GetAllKeptnResources(fs afero.Fs, resource string) ( return keptnResources, nil } +func (k *configServiceImpl) GetServiceResource(resource string, gitCommitID string) ([]byte, error) { + scope := api.NewResourceScope() + scope.Project(k.eventProperties.Project) + scope.Stage(k.eventProperties.Stage) + scope.Service(k.eventProperties.Service) + return k.fetchKeptnResource(resource, scope, gitCommitID) +} + +func (k *configServiceImpl) GetStageResource(resource string, gitCommitID string) ([]byte, error) { + scope := api.NewResourceScope() + scope.Project(k.eventProperties.Project) + scope.Stage(k.eventProperties.Stage) + return k.fetchKeptnResource(resource, scope, gitCommitID) +} + +func (k *configServiceImpl) GetProjectResource(resource string, gitCommitID string) ([]byte, error) { + scope := api.NewResourceScope() + scope.Project(k.eventProperties.Project) + return k.fetchKeptnResource(resource, scope, gitCommitID) +} + +func (k *configServiceImpl) GetJobConfiguration() (*config.Config, error) { + config, _, err := k.jobConfigReader.GetJobConfig(k.eventProperties.GitCommitID) + return config, err +} + /** * Retrieves a resource (=file) from the local file system. Basically checks if the file is available and if so returns it */ diff --git a/pkg/keptn/config_service_test.go b/pkg/keptn/config_service_test.go index 83bc61bd..56f3516d 100644 --- a/pkg/keptn/config_service_test.go +++ b/pkg/keptn/config_service_test.go @@ -1,6 +1,7 @@ package keptn import ( + "errors" "fmt" "net/url" "path" @@ -28,6 +29,10 @@ const service = "carts" const project = "sockshop" const stage = "dev" const gitCommitID = "6caf78d2c978f7f787" +const emptyJobConfig = `apiVersion: v2 +actions: + - name: Empty +` func TestGetAllKeptnResources(t *testing.T) { locustBasic := "/locust/basic.py" @@ -213,6 +218,73 @@ func TestErrorNoDirectoryResourcesLocal(t *testing.T) { require.Error(t, err) } +func TestConfigServiceImpl_GetJobConfiguration(t *testing.T) { + + serviceScope := api.NewResourceScope() + serviceScope.Project(project) + serviceScope.Stage(stage) + serviceScope.Service(service) + serviceScope.Resource(url.QueryEscape("job/config.yaml")) + + stageScope := api.NewResourceScope() + stageScope.Project(project) + stageScope.Stage(stage) + stageScope.Resource(url.QueryEscape("job/config.yaml")) + + projectScope := api.NewResourceScope() + projectScope.Project(project) + projectScope.Resource(url.QueryEscape("job/config.yaml")) + + event := EventProperties{ + Project: project, + Stage: stage, + Service: service, + GitCommitID: "", + } + + jobConfigResource := &models.Resource{ + Metadata: nil, + ResourceContent: emptyJobConfig, + ResourceURI: nil, + } + + t.Run("service configuration", func(t *testing.T) { + resourceHandlerMock := CreateResourceHandlerMock(t) + configService := NewConfigService(false, event, resourceHandlerMock) + + resourceHandlerMock.EXPECT().GetResource(gomock.Any(), *serviceScope, gomock.Any()).Return(jobConfigResource, nil) + + configuration, err := configService.GetJobConfiguration() + assert.NoError(t, err) + assert.Len(t, configuration.Actions, 1) + }) + + t.Run("stage configuration", func(t *testing.T) { + resourceHandlerMock := CreateResourceHandlerMock(t) + configService := NewConfigService(false, event, resourceHandlerMock) + + resourceHandlerMock.EXPECT().GetResource(gomock.Any(), *serviceScope, gomock.Any()).Return(nil, errors.New("error")) + resourceHandlerMock.EXPECT().GetResource(gomock.Any(), *stageScope, gomock.Any()).Return(jobConfigResource, nil) + + configuration, err := configService.GetJobConfiguration() + assert.NoError(t, err) + assert.Len(t, configuration.Actions, 1) + }) + + t.Run("project configuration", func(t *testing.T) { + resourceHandlerMock := CreateResourceHandlerMock(t) + configService := NewConfigService(false, event, resourceHandlerMock) + + resourceHandlerMock.EXPECT().GetResource(gomock.Any(), *serviceScope, gomock.Any()).Return(nil, errors.New("error")) + resourceHandlerMock.EXPECT().GetResource(gomock.Any(), *stageScope, gomock.Any()).Return(nil, errors.New("error")) + resourceHandlerMock.EXPECT().GetResource(gomock.Any(), *projectScope, gomock.Any()).Return(jobConfigResource, nil) + + configuration, err := configService.GetJobConfiguration() + assert.NoError(t, err) + assert.Len(t, configuration.Actions, 1) + }) +} + func createFile(fs afero.Fs, fileName string, content []byte) error { file, err := fs.Create(fileName) if err != nil { diff --git a/pkg/keptn/fake/config_service_mock.go b/pkg/keptn/fake/config_service_mock.go index cc1af966..1a8bbccb 100644 --- a/pkg/keptn/fake/config_service_mock.go +++ b/pkg/keptn/fake/config_service_mock.go @@ -6,6 +6,7 @@ package fake import ( context "context" + config "keptn-contrib/job-executor-service/pkg/config" reflect "reflect" gomock "github.com/golang/mock/gomock" @@ -52,6 +53,21 @@ func (mr *MockConfigServiceMockRecorder) GetAllKeptnResources(fs, resource inter return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllKeptnResources", reflect.TypeOf((*MockConfigService)(nil).GetAllKeptnResources), fs, resource) } +// GetJobConfiguration mocks base method. +func (m *MockConfigService) GetJobConfiguration() (*config.Config, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetJobConfiguration") + ret0, _ := ret[0].(*config.Config) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetJobConfiguration indicates an expected call of GetJobConfiguration. +func (mr *MockConfigServiceMockRecorder) GetJobConfiguration() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetJobConfiguration", reflect.TypeOf((*MockConfigService)(nil).GetJobConfiguration)) +} + // GetKeptnResource mocks base method. func (m *MockConfigService) GetKeptnResource(fs afero.Fs, resource string) ([]byte, error) { m.ctrl.T.Helper() diff --git a/pkg/keptn/fake/keptn_resourcehandler_mock.go b/pkg/keptn/fake/keptn_resourcehandler_mock.go index 9aa12edb..e00aafd7 100644 --- a/pkg/keptn/fake/keptn_resourcehandler_mock.go +++ b/pkg/keptn/fake/keptn_resourcehandler_mock.go @@ -12,31 +12,31 @@ import ( api "github.com/keptn/go-utils/pkg/api/utils" ) -// MockKeptnResourceHandler is a mock of KeptnResourceHandler interface. -type MockKeptnResourceHandler struct { +// MockV1KeptnResourceHandler is a mock of V1KeptnResourceHandler interface. +type MockV1KeptnResourceHandler struct { ctrl *gomock.Controller - recorder *MockKeptnResourceHandlerMockRecorder + recorder *MockV1KeptnResourceHandlerMockRecorder } -// MockKeptnResourceHandlerMockRecorder is the mock recorder for MockKeptnResourceHandler. -type MockKeptnResourceHandlerMockRecorder struct { - mock *MockKeptnResourceHandler +// MockV1KeptnResourceHandlerMockRecorder is the mock recorder for MockV1KeptnResourceHandler. +type MockV1KeptnResourceHandlerMockRecorder struct { + mock *MockV1KeptnResourceHandler } -// NewMockKeptnResourceHandler creates a new mock instance. -func NewMockKeptnResourceHandler(ctrl *gomock.Controller) *MockKeptnResourceHandler { - mock := &MockKeptnResourceHandler{ctrl: ctrl} - mock.recorder = &MockKeptnResourceHandlerMockRecorder{mock} +// NewMockV1KeptnResourceHandler creates a new mock instance. +func NewMockV1KeptnResourceHandler(ctrl *gomock.Controller) *MockV1KeptnResourceHandler { + mock := &MockV1KeptnResourceHandler{ctrl: ctrl} + mock.recorder = &MockV1KeptnResourceHandlerMockRecorder{mock} return mock } // EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockKeptnResourceHandler) EXPECT() *MockKeptnResourceHandlerMockRecorder { +func (m *MockV1KeptnResourceHandler) EXPECT() *MockV1KeptnResourceHandlerMockRecorder { return m.recorder } // GetResource mocks base method. -func (m *MockKeptnResourceHandler) GetResource(scope api.ResourceScope, options ...api.URIOption) (*models.Resource, error) { +func (m *MockV1KeptnResourceHandler) GetResource(scope api.ResourceScope, options ...api.URIOption) (*models.Resource, error) { m.ctrl.T.Helper() varargs := []interface{}{scope} for _, a := range options { @@ -49,8 +49,8 @@ func (m *MockKeptnResourceHandler) GetResource(scope api.ResourceScope, options } // GetResource indicates an expected call of GetResource. -func (mr *MockKeptnResourceHandlerMockRecorder) GetResource(scope interface{}, options ...interface{}) *gomock.Call { +func (mr *MockV1KeptnResourceHandlerMockRecorder) GetResource(scope interface{}, options ...interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() varargs := append([]interface{}{scope}, options...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResource", reflect.TypeOf((*MockKeptnResourceHandler)(nil).GetResource), varargs...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResource", reflect.TypeOf((*MockV1KeptnResourceHandler)(nil).GetResource), varargs...) } diff --git a/pkg/keptn/resource_service.go b/pkg/keptn/resource_service.go index 0da47590..2b80ac43 100644 --- a/pkg/keptn/resource_service.go +++ b/pkg/keptn/resource_service.go @@ -8,6 +8,13 @@ import ( "net/url" ) +// ResourceHandler is an interface that describes the functions for fetching resources from a service, stage or project level +type ResourceHandler interface { + GetServiceResource(resource string, gitCommitID string) ([]byte, error) + GetStageResource(resource string, gitCommitID string) ([]byte, error) + GetProjectResource(resource string, gitCommitID string) ([]byte, error) +} + // V1ResourceHandler is a wrapper around the v1 ResourceHandler of the Keptn API to simplify the // getting of resources of a given event type V1ResourceHandler struct { @@ -16,7 +23,7 @@ type V1ResourceHandler struct { } // NewV1ResourceHandler creates a new V1ResourceHandler from a given Keptn event and a V1KeptnResourceHandler -func NewV1ResourceHandler(event keptn.EventProperties, handler V1KeptnResourceHandler) V1ResourceHandler { +func NewV1ResourceHandler(event keptn.EventProperties, handler V1KeptnResourceHandler) ResourceHandler { return V1ResourceHandler{ Event: EventProperties{ Project: event.GetProject(), @@ -34,14 +41,8 @@ type V1KeptnResourceHandler interface { GetResource(scope api.ResourceScope, options ...api.URIOption) (*models.Resource, error) } -// GetResource returns the contents of a resource for a given gitCommitID -func (r V1ResourceHandler) GetResource(resource string, gitCommitID string) ([]byte, error) { - scope := api.NewResourceScope() - scope.Service(r.Event.Service) - scope.Project(r.Event.Project) - scope.Stage(r.Event.Stage) - scope.Resource(resource) - +// buildResourceHandlerV1Options builds the URIOption list such that it contains a well formatted gitCommitID +func buildResourceHandlerV1Options(gitCommitID string) api.URIOption { var queryParam api.URIOption if gitCommitID != "" { queryParam = api.AppendQuery(url.Values{ @@ -51,7 +52,47 @@ func (r V1ResourceHandler) GetResource(resource string, gitCommitID string) ([]b queryParam = api.AppendQuery(url.Values{}) } - resourceContent, err := r.ResourceHandler.GetResource(*scope, queryParam) + return queryParam +} + +// GetServiceResource returns the contents of a resource for a given gitCommitID +func (r V1ResourceHandler) GetServiceResource(resource string, gitCommitID string) ([]byte, error) { + scope := api.NewResourceScope() + scope.Service(r.Event.Service) + scope.Project(r.Event.Project) + scope.Stage(r.Event.Stage) + scope.Resource(resource) + + resourceContent, err := r.ResourceHandler.GetResource(*scope, buildResourceHandlerV1Options(gitCommitID)) + if err != nil { + return nil, fmt.Errorf("unable to get resouce from keptn: %w", err) + } + + return []byte(resourceContent.ResourceContent), nil +} + +// GetProjectResource returns the resource that was defined on project level +func (r V1ResourceHandler) GetProjectResource(resource string, gitCommitID string) ([]byte, error) { + scope := api.NewResourceScope() + scope.Project(r.Event.Project) + scope.Resource(resource) + + resourceContent, err := r.ResourceHandler.GetResource(*scope, buildResourceHandlerV1Options(gitCommitID)) + if err != nil { + return nil, fmt.Errorf("unable to get resouce from keptn: %w", err) + } + + return []byte(resourceContent.ResourceContent), nil +} + +// GetStageResource returns the resource that was defined in the stage +func (r V1ResourceHandler) GetStageResource(resource string, gitCommitID string) ([]byte, error) { + scope := api.NewResourceScope() + scope.Project(r.Event.Project) + scope.Stage(r.Event.Stage) + scope.Resource(resource) + + resourceContent, err := r.ResourceHandler.GetResource(*scope, buildResourceHandlerV1Options(gitCommitID)) if err != nil { return nil, fmt.Errorf("unable to get resouce from keptn: %w", err) } diff --git a/pkg/keptn/resource_service_test.go b/pkg/keptn/resource_service_test.go index c79b67c5..316e932a 100644 --- a/pkg/keptn/resource_service_test.go +++ b/pkg/keptn/resource_service_test.go @@ -13,7 +13,7 @@ func TestV1ResourceHandler_GetResource(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - mockResourceHandler := keptnfake.NewMockKeptnResourceHandler(mockCtrl) + mockResourceHandler := keptnfake.NewMockV1KeptnResourceHandler(mockCtrl) handler := V1ResourceHandler{ Event: EventProperties{ @@ -39,7 +39,7 @@ func TestV1ResourceHandler_GetResource(t *testing.T) { } for _, test := range tests { - t.Run(test.Test, func(t *testing.T) { + t.Run("GetServiceResource_"+test.Test, func(t *testing.T) { expectedBytes := []byte("") scope := api.NewResourceScope() @@ -54,7 +54,44 @@ func TestV1ResourceHandler_GetResource(t *testing.T) { ResourceURI: nil, }, nil) - resource, err := handler.GetResource("resource", test.GitCommitID) + resource, err := handler.GetServiceResource("resource", test.GitCommitID) + require.NoError(t, err) + require.Equal(t, expectedBytes, resource) + }) + + t.Run("GetStageResource_"+test.Test, func(t *testing.T) { + expectedBytes := []byte("") + + scope := api.NewResourceScope() + scope.Project("project") + scope.Resource("resource") + scope.Stage("stage") + + mockResourceHandler.EXPECT().GetResource(*scope, gomock.Len(1)).Times(1).Return(&models.Resource{ + Metadata: nil, + ResourceContent: string(expectedBytes), + ResourceURI: nil, + }, nil) + + resource, err := handler.GetStageResource("resource", test.GitCommitID) + require.NoError(t, err) + require.Equal(t, expectedBytes, resource) + }) + + t.Run("GetProjectResource_"+test.Test, func(t *testing.T) { + expectedBytes := []byte("") + + scope := api.NewResourceScope() + scope.Project("project") + scope.Resource("resource") + + mockResourceHandler.EXPECT().GetResource(*scope, gomock.Len(1)).Times(1).Return(&models.Resource{ + Metadata: nil, + ResourceContent: string(expectedBytes), + ResourceURI: nil, + }, nil) + + resource, err := handler.GetProjectResource("resource", test.GitCommitID) require.NoError(t, err) require.Equal(t, expectedBytes, resource) })