From c02c3d329cf988f279c839163e487d5a4c522d18 Mon Sep 17 00:00:00 2001 From: Vladislav Sukhin Date: Fri, 28 Jun 2024 20:44:56 +0300 Subject: [PATCH] feat: cd event test workflow atifact Signed-off-by: Vladislav Sukhin --- cmd/api-server/main.go | 1 + cmd/testworkflow-toolkit/artifacts/handler.go | 60 ++++++- .../commands/artifacts.go | 13 ++ cmd/testworkflow-toolkit/env/config.go | 2 + .../model_test_workflow_execution_extended.go | 22 +++ .../model_test_workflow_step_extended.go | 21 ++- ...el_test_workflow_step_parallel_extended.go | 15 ++ pkg/mapper/cdevents/mapper.go | 153 +++++++++--------- .../testworkflowexecutor/executor.go | 6 +- .../testworkflowprocessor/container.go | 2 + 10 files changed, 211 insertions(+), 84 deletions(-) diff --git a/cmd/api-server/main.go b/cmd/api-server/main.go index 2309693b589..69eb4a57c87 100644 --- a/cmd/api-server/main.go +++ b/cmd/api-server/main.go @@ -573,6 +573,7 @@ func main() { cfg.EnableImageDataPersistentCache, cfg.ImageDataPersistentCacheKey, cfg.TestkubeDashboardURI, + clusterId, ) go testWorkflowExecutor.Recover(context.Background()) diff --git a/cmd/testworkflow-toolkit/artifacts/handler.go b/cmd/testworkflow-toolkit/artifacts/handler.go index 1a635a35c6f..b1e17980539 100644 --- a/cmd/testworkflow-toolkit/artifacts/handler.go +++ b/cmd/testworkflow-toolkit/artifacts/handler.go @@ -1,21 +1,28 @@ package artifacts import ( + "context" "fmt" "io/fs" "path/filepath" "sync/atomic" + cdevents "github.com/cdevents/sdk-go/pkg/api" + cloudevents "github.com/cloudevents/sdk-go/v2" "github.com/dustin/go-humanize" + "github.com/gabriel-vasile/mimetype" + cde "github.com/kubeshop/testkube/pkg/mapper/cdevents" "github.com/kubeshop/testkube/pkg/ui" ) type handler struct { - uploader Uploader - processor Processor - postProcessor PostProcessor - pathPrefix string + uploader Uploader + processor Processor + postProcessor PostProcessor + pathPrefix string + cdeventsClient cloudevents.Client + cdeventsArtifactParameters cde.CDEventsArtifactParameters success atomic.Uint32 errors atomic.Uint32 @@ -42,6 +49,22 @@ func WithPathPrefix(pathPrefix string) HandlerOpts { } } +func WithCDEventsTarget(cdEventsTarget string) HandlerOpts { + return func(h *handler) { + var err error + h.cdeventsClient, err = cloudevents.NewClientHTTP(cloudevents.WithTarget(cdEventsTarget)) + if err != nil { + fmt.Printf(ui.LightYellow("failed to create cloud event client: %s"), err.Error()) + } + } +} + +func WithCDEventsArtifactParameters(cdeventsArtifactParameters cde.CDEventsArtifactParameters) HandlerOpts { + return func(h *handler) { + h.cdeventsArtifactParameters = cdeventsArtifactParameters + } +} + func NewHandler(uploader Uploader, processor Processor, opts ...HandlerOpts) Handler { h := &handler{ uploader: uploader, @@ -82,6 +105,12 @@ func (h *handler) Add(path string, file fs.File, stat fs.FileInfo) (err error) { err = h.processor.Add(h.uploader, uploadPath, file, stat) if err == nil { h.success.Add(1) + if h.cdeventsClient != nil { + err = h.sendCDEvent(path) + if err != nil { + fmt.Printf(ui.LightYellow("failed to send cd event: %s"), err.Error()) + } + } } else { h.errors.Add(1) fmt.Printf(ui.Red("%s: failed: %s"), uploadPath, err.Error()) @@ -128,3 +157,26 @@ func (h *handler) End() (err error) { } return nil } + +func (h *handler) sendCDEvent(path string) error { + mtype, err := mimetype.DetectFile(path) + if err != nil { + return err + } + + ev, err := cde.MapTestkubeTestWorkflowArtifactToCDEvent(h.cdeventsArtifactParameters, path, mtype.String()) + if err != nil { + return err + } + + ce, err := cdevents.AsCloudEvent(ev) + if err != nil { + return err + } + + if result := h.cdeventsClient.Send(context.Background(), *ce); cloudevents.IsUndelivered(result) { + return fmt.Errorf("failed to send, %v", result) + } + + return nil +} diff --git a/cmd/testworkflow-toolkit/commands/artifacts.go b/cmd/testworkflow-toolkit/commands/artifacts.go index ae4bc854259..d047aa01598 100644 --- a/cmd/testworkflow-toolkit/commands/artifacts.go +++ b/cmd/testworkflow-toolkit/commands/artifacts.go @@ -23,6 +23,7 @@ import ( "github.com/kubeshop/testkube/cmd/testworkflow-toolkit/artifacts" "github.com/kubeshop/testkube/cmd/testworkflow-toolkit/env" + "github.com/kubeshop/testkube/pkg/mapper/cdevents" "github.com/kubeshop/testkube/pkg/ui" ) @@ -146,6 +147,18 @@ func NewArtifactsCmd() *cobra.Command { handlerOpts = append(handlerOpts, artifacts.WithPathPrefix(env.Config().Execution.FSPrefix)) } + // Support cd evaents + if env.Config().System.CDEventTarget != "" { + handlerOpts = append(handlerOpts, artifacts.WithCDEventsTarget(env.Config().System.CDEventTarget)) + handlerOpts = append(handlerOpts, artifacts.WithCDEventsArtifactParameters(cdevents.CDEventsArtifactParameters{ + Id: env.Config().Execution.Id, + Name: env.Config().Execution.Name, + WorkflowName: env.Config().Execution.WorkflowName, + ClusterID: env.Config().System.ClusterID, + DashboardURI: env.Config().System.DashboardUrl, + })) + } + handler := artifacts.NewHandler(uploader, processor, handlerOpts...) run(handler, walker, os.DirFS("/")) diff --git a/cmd/testworkflow-toolkit/env/config.go b/cmd/testworkflow-toolkit/env/config.go index ed174880c64..31851130408 100644 --- a/cmd/testworkflow-toolkit/env/config.go +++ b/cmd/testworkflow-toolkit/env/config.go @@ -54,6 +54,8 @@ type envSystemConfig struct { Ip string `envconfig:"TK_IP"` DashboardUrl string `envconfig:"TK_DASH"` ApiUrl string `envconfig:"TK_API"` + ClusterID string `envconfig:"TK_CLU"` + CDEventTarget string `envconfig:"TK_CDE"` } type envImagesConfig struct { diff --git a/pkg/api/v1/testkube/model_test_workflow_execution_extended.go b/pkg/api/v1/testkube/model_test_workflow_execution_extended.go index a50732cf9bb..66a9b90afc7 100644 --- a/pkg/api/v1/testkube/model_test_workflow_execution_extended.go +++ b/pkg/api/v1/testkube/model_test_workflow_execution_extended.go @@ -47,6 +47,10 @@ func (e *TestWorkflowExecution) GetNamespace(defaultNamespace string) string { } func (e *TestWorkflowExecution) ContainsExecuteAction() bool { + if e == nil { + return false + } + if e.ResolvedWorkflow == nil || e.ResolvedWorkflow.Spec == nil { return false } @@ -60,3 +64,21 @@ func (e *TestWorkflowExecution) ContainsExecuteAction() bool { return false } + +func (e *TestWorkflowExecution) GetTemplateRefs() []TestWorkflowTemplateRef { + if e == nil { + return nil + } + + if e.ResolvedWorkflow == nil || e.ResolvedWorkflow.Spec == nil { + return nil + } + + var templateRefs []TestWorkflowTemplateRef + steps := append(e.ResolvedWorkflow.Spec.Setup, append(e.ResolvedWorkflow.Spec.Steps, e.ResolvedWorkflow.Spec.After...)...) + for _, step := range steps { + templateRefs = append(templateRefs, step.GetTemplateRefs()...) + } + + return templateRefs +} diff --git a/pkg/api/v1/testkube/model_test_workflow_step_extended.go b/pkg/api/v1/testkube/model_test_workflow_step_extended.go index 951e91dc1e6..b8ddb9db83e 100644 --- a/pkg/api/v1/testkube/model_test_workflow_step_extended.go +++ b/pkg/api/v1/testkube/model_test_workflow_step_extended.go @@ -40,9 +40,28 @@ func (w *TestWorkflowStep) ContainsExecuteAction() bool { } } - if w.Parallel.ContainsExecuteAction() { + if w.Parallel != nil && w.Parallel.ContainsExecuteAction() { return true } return false } + +func (w *TestWorkflowStep) GetTemplateRefs() []TestWorkflowTemplateRef { + var templateRefs []TestWorkflowTemplateRef + + if w.Template != nil { + templateRefs = append(templateRefs, *w.Template) + } + + steps := append(w.Setup, w.Steps...) + for _, step := range steps { + templateRefs = append(templateRefs, step.GetTemplateRefs()...) + } + + if w.Parallel != nil { + templateRefs = append(templateRefs, w.Parallel.GetTemplateRefs()...) + } + + return templateRefs +} diff --git a/pkg/api/v1/testkube/model_test_workflow_step_parallel_extended.go b/pkg/api/v1/testkube/model_test_workflow_step_parallel_extended.go index d4f8f785aff..7ad342975e1 100644 --- a/pkg/api/v1/testkube/model_test_workflow_step_parallel_extended.go +++ b/pkg/api/v1/testkube/model_test_workflow_step_parallel_extended.go @@ -14,3 +14,18 @@ func (w *TestWorkflowStepParallel) ContainsExecuteAction() bool { return false } + +func (w *TestWorkflowStepParallel) GetTemplateRefs() []TestWorkflowTemplateRef { + var templateRefs []TestWorkflowTemplateRef + + if w.Template != nil { + templateRefs = append(templateRefs, *w.Template) + } + + steps := append(w.Setup, append(w.Steps, w.After...)...) + for _, step := range steps { + templateRefs = append(templateRefs, step.GetTemplateRefs()...) + } + + return templateRefs +} diff --git a/pkg/mapper/cdevents/mapper.go b/pkg/mapper/cdevents/mapper.go index ce3e2658c04..a205f706d59 100644 --- a/pkg/mapper/cdevents/mapper.go +++ b/pkg/mapper/cdevents/mapper.go @@ -27,34 +27,19 @@ func MapTestkubeEventToCDEvent(tkEvent testkube.Event, clusterID, defaultNamespa case *testkube.EventEndTestSuiteAborted, *testkube.EventEndTestSuiteFailed, *testkube.EventEndTestSuiteTimeout, *testkube.EventEndTestSuiteSuccess: return MapTestkubeEventFinishTestSuiteToCDEvent(tkEvent, clusterID, dashboardURI) case *testkube.EventQueueTestWorkflow: - containsExuctionAction := false - if tkEvent.TestWorkflowExecution != nil { - containsExuctionAction = tkEvent.TestWorkflowExecution.ContainsExecuteAction() - } - - if containsExuctionAction { + if tkEvent.TestWorkflowExecution.ContainsExecuteAction() { return MapTestkubeEventQueuedTestWorkflowTestSuiteToCDEvent(tkEvent, clusterID, defaultNamespace, dashboardURI) } return MapTestkubeEventQueuedTestWorkflowTestToCDEvent(tkEvent, clusterID, defaultNamespace, dashboardURI) case *testkube.EventStartTestWorkflow: - containsExuctionAction := false - if tkEvent.TestWorkflowExecution != nil { - containsExuctionAction = tkEvent.TestWorkflowExecution.ContainsExecuteAction() - } - - if containsExuctionAction { + if tkEvent.TestWorkflowExecution.ContainsExecuteAction() { return MapTestkubeEventStartTestWorkflowTestSuiteToCDEvent(tkEvent, clusterID, defaultNamespace, dashboardURI) } return MapTestkubeEventStartTestWorkflowTestToCDEvent(tkEvent, clusterID, defaultNamespace, dashboardURI) case *testkube.EventEndTestWorkflowAborted, *testkube.EventEndTestWorkflowFailed, *testkube.EventEndTestWorkflowSuccess: - containsExuctionAction := false - if tkEvent.TestWorkflowExecution != nil { - containsExuctionAction = tkEvent.TestWorkflowExecution.ContainsExecuteAction() - } - - if containsExuctionAction { + if tkEvent.TestWorkflowExecution.ContainsExecuteAction() { return MapTestkubeEventFinishTestWorkflowTestSuiteToCDEvent(tkEvent, clusterID, defaultNamespace, dashboardURI) } @@ -523,9 +508,9 @@ func MapTestkubeEventQueuedTestWorkflowTestToCDEvent(event testkube.Event, clust } ev.SetSubjectTestCase(&cdevents.TestCaseRunQueuedSubjectContentTestCase{ - Id: workflowName, - // Type: MapTestkubeTestTypeToCDEventTestCaseType(event.TestWorkflowExecution.TestType), - Uri: fmt.Sprintf("%s/test-workflows/%s", dashboardURI, workflowName), + Id: workflowName, + Type: MapTestkubeTestWorkflowTemplateToCDEventTestCaseType(event.TestWorkflowExecution.GetTemplateRefs()), + Uri: fmt.Sprintf("%s/test-workflows/%s", dashboardURI, workflowName), }) namespace := event.TestWorkflowExecution.Namespace @@ -537,20 +522,6 @@ func MapTestkubeEventQueuedTestWorkflowTestToCDEvent(event testkube.Event, clust Id: namespace, Source: clusterID, }) - /* - if event.TestWorkflowExecution.RunningContext != nil { - ev.SetSubjectTrigger(&cdevents.TestCaseRunQueuedSubjectContentTrigger{ - Type: MapTestkubeRunningContextTypeToCDEventTiggerType(event.TestWorkflowExecution.RunningContext.Type_), - }) - } - - if event.TestWorkflowExecution.ParentName != "" { - ev.SetSubjectTestSuiteRun(&cdevents.Reference{ - Id: event.TestWorkflowExecution.ParentName, - Source: clusterID, - }) - } - */ } return ev, nil @@ -590,13 +561,6 @@ func MapTestkubeEventQueuedTestWorkflowTestSuiteToCDEvent(event testkube.Event, Id: namespace, Source: clusterID, }) - /* - if event.TestWorkflowExecution.RunningContext != nil { - ev.SetSubjectTrigger(&cdevents.TestSuiteRunQueuedSubjectContentTrigger{ - Type: MapTestkubeRunningContextTypeToCDEventTiggerType(event.TestWorkflowExecution.RunningContext.Type_), - }) - } - */ } return ev, nil @@ -623,9 +587,9 @@ func MapTestkubeEventStartTestWorkflowTestToCDEvent(event testkube.Event, cluste } ev.SetSubjectTestCase(&cdevents.TestCaseRunStartedSubjectContentTestCase{ - Id: workflowName, - // Type: MapTestkubeTestTypeToCDEventTestCaseType(event.TestWorkflowExecution.TestType), - Uri: fmt.Sprintf("%s/test-workflows/%s", dashboardURI, workflowName), + Id: workflowName, + Type: MapTestkubeTestWorkflowTemplateToCDEventTestCaseType(event.TestWorkflowExecution.GetTemplateRefs()), + Uri: fmt.Sprintf("%s/test-workflows/%s", dashboardURI, workflowName), }) namespace := event.TestWorkflowExecution.Namespace @@ -637,20 +601,6 @@ func MapTestkubeEventStartTestWorkflowTestToCDEvent(event testkube.Event, cluste Id: namespace, Source: clusterID, }) - /* - if event.TestWorkflowExecution.RunningContext != nil { - ev.SetSubjectTrigger(&cdevents.TestCaseRunStartedSubjectContentTrigger{ - Type: MapTestkubeRunningContextTypeToCDEventTiggerType(event.TestWorkflowExecution.RunningContext.Type_), - }) - } - - if event.TestWorkflowExecution.ParentName != "" { - ev.SetSubjectTestSuiteRun(&cdevents.Reference{ - Id: event.TestWorkflowExecution.ParentName, - Source: clusterID, - }) - } - */ } return ev, nil @@ -690,13 +640,6 @@ func MapTestkubeEventStartTestWorkflowTestSuiteToCDEvent(event testkube.Event, c Id: namespace, Source: clusterID, }) - /* - if event.TestWorkflowExecution.RunningContext != nil { - ev.SetSubjectTrigger(&cdevents.TestSuiteRunStartedSubjectContentTrigger{ - Type: MapTestkubeRunningContextTypeToCDEventTiggerType(event.TestWorkflowExecution.RunningContext.Type_), - }) - } - */ } return ev, nil @@ -723,9 +666,9 @@ func MapTestkubeEventFinishTestWorkflowTestToCDEvent(event testkube.Event, clust } ev.SetSubjectTestCase(&cdevents.TestCaseRunFinishedSubjectContentTestCase{ - Id: workflowName, - // Type: MapTestkubeTestTypeToCDEventTestCaseType(event.TestWokflowExecution.TestType), - Uri: fmt.Sprintf("%s/test-workflows/%s", dashboardURI, workflowName), + Id: workflowName, + Type: MapTestkubeTestWorkflowTemplateToCDEventTestCaseType(event.TestWorkflowExecution.GetTemplateRefs()), + Uri: fmt.Sprintf("%s/test-workflows/%s", dashboardURI, workflowName), }) namespace := event.TestWorkflowExecution.Namespace @@ -765,14 +708,6 @@ func MapTestkubeEventFinishTestWorkflowTestToCDEvent(event testkube.Event, clust ev.SetSubjectOutcome("pass") } } - /* - if event.TestWorkflowExecution.ParentName != "" { - ev.SetSubjectTestSuiteRun(&cdevents.Reference{ - Id: event.TestWorkflowExecution.ParentName, - Source: clusterID, - }) - } - */ } return ev, nil @@ -875,9 +810,71 @@ func MapTestkubeTestWorkflowLogToCDEvent(event testkube.Event, clusterID, dashbo workflowName = event.TestWorkflowExecution.Workflow.Name } - ev.SetSubjectUri(fmt.Sprintf("%s/test-workflows/%s/executions/%s/log-output", dashboardURI, + ev.SetSubjectUri(fmt.Sprintf("%s/test-workflows/%s/executions/%s", dashboardURI, workflowName, event.TestWorkflowExecution.Id)) } return ev, nil } + +// MapTestkubeTestWorkflowTemplateToCDEventTestCaseType maps OpenAPI spec Test Workflow Template to CDEvent Test Case Type +func MapTestkubeTestWorkflowTemplateToCDEventTestCaseType(templateRefs []testkube.TestWorkflowTemplateRef) string { + var types = map[string]string{ + "official--artillery": "performance", + "official--cypress": "functional", + "official--gradle": "integration", + "official--jmeter": "performance", + "official--k6": "performance", + "official--maven": "integration", + "official--playwright": "functional", + "official--postman": "functional", + } + + templateNames := make(map[string]struct{}) + for _, templateRef := range templateRefs { + if strings.Contains(templateRef.Name, "official--") { + templateNames[templateRef.Name] = struct{}{} + } + } + + for key, value := range types { + for templateName := range templateNames { + if strings.Contains(templateName, key) { + return value + } + } + } + + return "other" +} + +// CDEventsArtifactParameters contains cd events artifact parameters +type CDEventsArtifactParameters struct { + Id string + Name string + WorkflowName string + ClusterID string + DashboardURI string +} + +// MapTestkubeGTestWorkflowArtifactToCDEvent maps OpenAPI spec Test Artifact to CDEvent CDEventReader +func MapTestkubeTestWorkflowArtifactToCDEvent(parameters CDEventsArtifactParameters, path, format string) (cdevents.CDEventReader, error) { + // Create the base event + ev, err := cdevents.NewTestOutputPublishedEvent() + if err != nil { + return nil, err + } + + ev.SetSubjectId(filepath.Join(parameters.Name, path)) + ev.SetSubjectSource(parameters.ClusterID) + ev.SetSource(parameters.ClusterID) + ev.SetSubjectTestCaseRun(&cdevents.Reference{ + Id: parameters.Id, + Source: parameters.ClusterID, + }) + + ev.SetSubjectFormat(format) + ev.SetSubjectOutputType(MapMimeTypeToCDEventOutputType(format)) + ev.SetSubjectUri(fmt.Sprintf("%s/test-workflows/%s/overview/%s/artifacts", parameters.DashboardURI, parameters.WorkflowName, parameters.Id)) + return ev, nil +} diff --git a/pkg/testworkflows/testworkflowexecutor/executor.go b/pkg/testworkflows/testworkflowexecutor/executor.go index 94c99d2bfec..2dc2c222614 100644 --- a/pkg/testworkflows/testworkflowexecutor/executor.go +++ b/pkg/testworkflows/testworkflowexecutor/executor.go @@ -65,6 +65,7 @@ type executor struct { enableImageDataPersistentCache bool imageDataPersistentCacheKey string dashboardURI string + clusterID string serviceAccountNames map[string]string } @@ -81,7 +82,7 @@ func New(emitter *event.Emitter, metrics v1.Metrics, serviceAccountNames map[string]string, globalTemplateName, namespace, apiUrl, defaultRegistry string, - enableImageDataPersistentCache bool, imageDataPersistentCacheKey, dashboardURI string) TestWorkflowExecutor { + enableImageDataPersistentCache bool, imageDataPersistentCacheKey, dashboardURI, clusterID string) TestWorkflowExecutor { if serviceAccountNames == nil { serviceAccountNames = make(map[string]string) } @@ -106,6 +107,7 @@ func New(emitter *event.Emitter, enableImageDataPersistentCache: enableImageDataPersistentCache, imageDataPersistentCacheKey: imageDataPersistentCacheKey, dashboardURI: dashboardURI, + clusterID: clusterID, } } @@ -425,6 +427,8 @@ func (e *executor) Execute(ctx context.Context, workflow testworkflowsv1.TestWor "api.url": e.apiUrl, "namespace": namespace, "defaultRegistry": e.defaultRegistry, + "clusterId": e.clusterID, + "cdeventsTarget": os.Getenv("CDEVENTS_TARGET"), "images.init": constants.DefaultInitImage, "images.toolkit": constants.DefaultToolkitImage, diff --git a/pkg/testworkflows/testworkflowprocessor/container.go b/pkg/testworkflows/testworkflowprocessor/container.go index 51afcb64548..3b2a01de28b 100644 --- a/pkg/testworkflows/testworkflowprocessor/container.go +++ b/pkg/testworkflows/testworkflowprocessor/container.go @@ -431,6 +431,8 @@ func (c *container) EnableToolkit(ref string) Container { "TK_SA": "{{internal.serviceaccount.default}}", "TK_DASH": "{{internal.dashboard.url}}", "TK_API": "{{internal.api.url}}", + "TK_CLU": "{{internal.clusterId}}", + "TK_CDE": "{{internal.cdeventsTarget}}", "TK_C_URL": "{{internal.cloud.api.url}}", "TK_C_KEY": "{{internal.cloud.api.key}}", "TK_C_TLS_INSECURE": "{{internal.cloud.api.tlsInsecure}}",