feat: support inlined global template in the runner (#6137)

kubeshop · Jan 28, 2025 · 3a76f00 · 3a76f00
1 parent 84658dc
commit 3a76f00
Show file tree

Hide file tree

Showing 7 changed files with 67 additions and 31 deletions.
diff --git a/cmd/api-server/main.go b/cmd/api-server/main.go
@@ -241,11 +241,12 @@ func main() {
 		proContext,
 		executionWorker,
 		runner2.Options{
-			ClusterID:           clusterId,
-			DashboardURI:        cfg.TestkubeDashboardURI,
-			DefaultNamespace:    cfg.TestkubeNamespace,
-			ServiceAccountNames: serviceAccountNames,
-			StorageSkipVerify:   cfg.StorageSkipVerify,
+			ClusterID:            clusterId,
+			DashboardURI:         cfg.TestkubeDashboardURI,
+			DefaultNamespace:     cfg.TestkubeNamespace,
+			ServiceAccountNames:  serviceAccountNames,
+			StorageSkipVerify:    cfg.StorageSkipVerify,
+			GlobalTemplateInline: cfg.GlobalWorkflowTemplateInline,
 		},
 	)
 	if !cfg.DisableRunner {
@@ -268,7 +269,6 @@ func main() {
 		metrics,
 		secretManager,
 		cfg.GlobalWorkflowTemplateName,
-		cfg.GlobalWorkflowTemplateInline,
 		cfg.TestkubeDashboardURI,
 		proContext.OrgID,
 		proContext.EnvID,

diff --git a/cmd/api-server/services/controlplane.go b/cmd/api-server/services/controlplane.go
@@ -405,7 +405,6 @@ func CreateControlPlane(ctx context.Context, cfg *config.Config, features featur
 		metrics,
 		secretManager,
 		cfg.GlobalWorkflowTemplateName,
-		cfg.GlobalWorkflowTemplateInline,
 		cfg.TestkubeDashboardURI,
 		"",
 		"",

diff --git a/cmd/tcl/kubectl-testkube/devbox/devutils/runneragent.go b/cmd/tcl/kubectl-testkube/devbox/devutils/runneragent.go
@@ -60,7 +60,7 @@ func (r *RunnerAgent) Create(ctx context.Context, env *client.Environment, runne
 		{Name: "TESTKUBE_PRO_TLS_INSECURE", Value: fmt.Sprintf("%v", r.cloud.AgentInsecure())},
 		{Name: "TESTKUBE_PRO_TLS_SKIP_VERIFY", Value: "true"},
 
-		// RunnerAgent configuration
+		// Runner configuration
 		{Name: "APISERVER_FULLNAME", Value: "devbox-agent"},
 		{Name: "TESTKUBE_NAMESPACE", Value: r.pod.Namespace()},
 		{Name: "JOB_SERVICE_ACCOUNT_NAME", Value: "devbox-account"},

diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go
@@ -2,20 +2,25 @@ package runner
 
 import (
 	"context"
+	"strings"
 	"sync"
 	"time"
 
 	"github.com/pkg/errors"
 
+	testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1"
 	"github.com/kubeshop/testkube/internal/app/api/metrics"
 	"github.com/kubeshop/testkube/internal/config"
+	"github.com/kubeshop/testkube/internal/crdcommon"
 	"github.com/kubeshop/testkube/pkg/api/v1/testkube"
 	"github.com/kubeshop/testkube/pkg/controlplaneclient"
 	"github.com/kubeshop/testkube/pkg/event"
+	"github.com/kubeshop/testkube/pkg/expressions"
 	"github.com/kubeshop/testkube/pkg/log"
 	configRepo "github.com/kubeshop/testkube/pkg/repository/config"
 	"github.com/kubeshop/testkube/pkg/testworkflows/executionworker/executionworkertypes"
 	"github.com/kubeshop/testkube/pkg/testworkflows/executionworker/registry"
+	"github.com/kubeshop/testkube/pkg/testworkflows/testworkflowresolver"
 )
 
 const (
@@ -24,6 +29,8 @@ const (
 
 	SaveEndResultRetryCount     = 100
 	SaveEndResultRetryBaseDelay = 500 * time.Millisecond
+
+	inlinedGlobalTemplateName = "<inline-global-template>"
 )
 
 type RunnerExecute interface {
@@ -41,14 +48,15 @@ type Runner interface {
 }
 
 type runner struct {
-	worker            executionworkertypes.Worker
-	client            controlplaneclient.Client
-	configRepository  configRepo.Repository
-	emitter           event.Interface
-	metrics           metrics.Metrics
-	proContext        config.ProContext // TODO: Include Agent ID in pro context
-	dashboardURI      string
-	storageSkipVerify bool
+	worker               executionworkertypes.Worker
+	client               controlplaneclient.Client
+	configRepository     configRepo.Repository
+	emitter              event.Interface
+	metrics              metrics.Metrics
+	proContext           config.ProContext // TODO: Include Agent ID in pro context
+	dashboardURI         string
+	storageSkipVerify    bool
+	globalTemplateInline *testworkflowsv1.TestWorkflowTemplate
 
 	watching sync.Map
 }
@@ -62,16 +70,29 @@ func New(
 	proContext config.ProContext,
 	dashboardURI string,
 	storageSkipVerify bool,
+	globalTemplateInlineYaml string,
 ) Runner {
+
+	var globalTemplateInline *testworkflowsv1.TestWorkflowTemplate
+	if globalTemplateInlineYaml != "" {
+		globalTemplateInline = new(testworkflowsv1.TestWorkflowTemplate)
+		err := crdcommon.DeserializeCRD(globalTemplateInline, []byte("spec:\n  "+strings.ReplaceAll(globalTemplateInlineYaml, "\n", "\n  ")))
+		globalTemplateInline.Name = inlinedGlobalTemplateName
+		if err != nil {
+			log.DefaultLogger.Errorw("failed to unmarshal inlined global template", "error", err)
+			globalTemplateInline = nil
+		}
+	}
 	return &runner{
-		worker:            worker,
-		configRepository:  configRepository,
-		client:            client,
-		emitter:           emitter,
-		metrics:           metrics,
-		proContext:        proContext,
-		dashboardURI:      dashboardURI,
-		storageSkipVerify: storageSkipVerify,
+		worker:               worker,
+		configRepository:     configRepository,
+		client:               client,
+		emitter:              emitter,
+		metrics:              metrics,
+		proContext:           proContext,
+		dashboardURI:         dashboardURI,
+		storageSkipVerify:    storageSkipVerify,
+		globalTemplateInline: globalTemplateInline,
 	}
 }
 
@@ -233,6 +254,19 @@ func (r *runner) Notifications(ctx context.Context, id string) executionworkerty
 }
 
 func (r *runner) Execute(request executionworkertypes.ExecuteRequest) (*executionworkertypes.ExecuteResult, error) {
+	if r.globalTemplateInline != nil {
+		testworkflowresolver.AddGlobalTemplateRef(&request.Workflow, testworkflowsv1.TemplateRef{
+			Name: testworkflowresolver.GetDisplayTemplateName(inlinedGlobalTemplateName),
+		})
+		err := testworkflowresolver.ApplyTemplates(&request.Workflow, map[string]*testworkflowsv1.TestWorkflowTemplate{
+			inlinedGlobalTemplateName: r.globalTemplateInline,
+		}, func(key, value string) (expressions.Expression, error) {
+			return nil, errors.New("not supported")
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
 	res, err := r.worker.Execute(context.Background(), request)
 	if err == nil {
 		// TODO: consider retry?

diff --git a/pkg/runner/service.go b/pkg/runner/service.go
@@ -23,6 +23,8 @@ type Options struct {
 	ServiceAccountNames map[string]string
 
 	StorageSkipVerify bool
+
+	GlobalTemplateInline string
 }
 
 type service struct {
@@ -69,6 +71,7 @@ func NewService(
 			proContext,
 			opts.DashboardURI,
 			opts.StorageSkipVerify,
+			opts.GlobalTemplateInline,
 		),
 	}
 }

diff --git a/pkg/testworkflows/testworkflowexecutor/executor.go b/pkg/testworkflows/testworkflowexecutor/executor.go
@@ -72,7 +72,6 @@ func New(
 	metrics v1.Metrics,
 	secretManager secretmanager.SecretManager,
 	globalTemplateName string,
-	globalTemplateInlineYaml string,
 	dashboardURI string,
 	organizationId string,
 	defaultEnvironmentId string,
@@ -100,7 +99,7 @@ func New(
 				return nil, nil
 			},
 			globalTemplateName,
-			globalTemplateInlineYaml,
+			"",
 			organizationId,
 			defaultEnvironmentId,
 			agentId,

diff --git a/pkg/testworkflows/testworkflowexecutor/scheduler.go b/pkg/testworkflows/testworkflowexecutor/scheduler.go
@@ -8,6 +8,7 @@ import (
 	"maps"
 	"math"
 	"slices"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -16,11 +17,10 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/encoding/gzip"
 	"google.golang.org/grpc/metadata"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/yaml"
 
 	testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1"
 	"github.com/kubeshop/testkube/internal/common"
+	"github.com/kubeshop/testkube/internal/crdcommon"
 	"github.com/kubeshop/testkube/pkg/api/v1/testkube"
 	"github.com/kubeshop/testkube/pkg/cloud"
 	"github.com/kubeshop/testkube/pkg/log"
@@ -81,12 +81,13 @@ func NewScheduler(
 ) Scheduler {
 	var globalTemplateInline *testkube.TestWorkflowTemplate
 	if globalTemplateInlineYaml != "" {
-		inline := &testworkflowsv1.TestWorkflowTemplate{ObjectMeta: metav1.ObjectMeta{Name: inlinedGlobalTemplateName}}
-		err := yaml.Unmarshal([]byte(globalTemplateInlineYaml), inline.Spec)
+		inline := new(testworkflowsv1.TestWorkflowTemplate)
+		err := crdcommon.DeserializeCRD(inline, []byte("spec:\n  "+strings.ReplaceAll(globalTemplateInlineYaml, "\n", "\n  ")))
+		inline.Name = inlinedGlobalTemplateName
 		if err == nil {
 			globalTemplateInline = testworkflows2.MapTemplateKubeToAPI(inline)
 		} else {
-			log.DefaultLogger.Errorw("failed to unmarshal inline global template", "error", err)
+			log.DefaultLogger.Errorw("failed to unmarshal inlined global template", "error", err)
 		}
 	}
 	return &scheduler{