feat: support Runner Agents (#6119)

* support new architecture - events propagated through Control Plane
* support new scheduling - executions scheduled via Control Plane
* support Runner Agent - agent that runs Test Workflows
* support GitOps Agent - agent that synchronises resources between Kubernetes and Control Plane
* support Cloud Storage - storing the resources in Control Plane instead of Kubernetes
* support inlined Global Templates - to avoid storing Global Template in Kubernetes

Makefile

@@ -64,6 +64,9 @@ build: build-api-server build-testkube-bin
 build-api-server:
 	go build -o $(BIN_DIR)/api-server -ldflags='$(LD_FLAGS)' cmd/api-server/main.go
 
+build-toolkit:
+	go build -o build/testworkflow-toolkit/testworkflow-toolkit -ldflags='$(LD_FLAGS)' cmd/testworkflow-toolkit/main.go
+
 build-testkube-bin:
 	go build \
 		-ldflags="-s -w -X main.version=999.0.0-$(COMMIT) \

api/v1/testkube.yaml

@@ -7712,6 +7712,8 @@ components:
             type: string
           example:
             WEBHOOK_PARAMETER: "any value"
+        external:
+          type: boolean
 
     EventResource:
       type: string
@@ -8437,6 +8439,10 @@ components:
           type: string
           format: date-time
           example: "2022-07-30T06:54:15Z"
+        updated:
+          type: string
+          format: date-time
+          example: "2022-07-30T06:54:15Z"
         spec:
           $ref: "#/components/schemas/TestWorkflowSpec"
         readOnly:
@@ -8462,14 +8468,39 @@ components:
           default: false
         tags:
           $ref: "#/components/schemas/TestWorkflowTagValue"
+        target:
+          $ref: "#/components/schemas/ExecutionTarget"
         runningContext:
           description: running context for the test workflow execution (Pro edition only)
           $ref: "#/components/schemas/TestWorkflowRunningContext"
         parentExecutionIds:
           type: array
           description: parent execution ids
           items:
-            type: string          
+            type: string
+
+    ExecutionTarget:
+      type: object
+      properties:
+        match:
+          type: object
+          description: runner labels to match
+          additionalProperties:
+            type: array
+            items:
+              type: string
+        not:
+          type: object
+          description: runner labels to NOT match
+          additionalProperties:
+            type: array
+            items:
+              type: string
+        replicate:
+          type: array
+          description: list of runner labels to replicate the executions
+          items:
+            type: string
 
     TestWorkflowWithExecution:
       type: object
@@ -8516,6 +8547,13 @@ components:
           description: identifier for group of correlated executions
           format: bson objectId
           example: "62f395e004109209b50edfc1"
+        runnerId:
+          type: string
+          description: identifier of the runner where it has been executed
+        runnerTarget:
+          $ref: "#/components/schemas/ExecutionTarget"
+        runnerOriginalTarget:
+          $ref: "#/components/schemas/ExecutionTarget"
         name:
           type: string
           description: execution name
@@ -8586,6 +8624,11 @@ components:
           description: unique execution identifier
           format: bson objectId
           example: "62f395e004109209b50edfc1"
+        groupId:
+          type: string
+          description: identifier for group of correlated executions
+          format: bson objectId
+          example: "62f395e004109209b50edfc1"
         name:
           type: string
           description: execution name
@@ -8922,6 +8965,10 @@ components:
           type: string
           format: date-time
           example: "2022-07-30T06:54:15Z"
+        updated:
+          type: string
+          format: date-time
+          example: "2022-07-30T06:54:15Z"
         spec:
           $ref: "#/components/schemas/TestWorkflowTemplateSpec"
 

cmd/api-server/commons/commons.go

@@ -23,7 +23,10 @@ import (
 	parser "github.com/kubeshop/testkube/internal/template"
 	"github.com/kubeshop/testkube/pkg/api/v1/testkube"
 	"github.com/kubeshop/testkube/pkg/cache"
+	"github.com/kubeshop/testkube/pkg/capabilities"
 	"github.com/kubeshop/testkube/pkg/cloud"
+	cloudconfig "github.com/kubeshop/testkube/pkg/cloud/data/config"
+	"github.com/kubeshop/testkube/pkg/cloud/data/executor"
 	"github.com/kubeshop/testkube/pkg/configmap"
 	"github.com/kubeshop/testkube/pkg/dbmigrator"
 	"github.com/kubeshop/testkube/pkg/event"
@@ -38,6 +41,7 @@ import (
 	"github.com/kubeshop/testkube/pkg/secret"
 	domainstorage "github.com/kubeshop/testkube/pkg/storage"
 	"github.com/kubeshop/testkube/pkg/storage/minio"
+	"github.com/kubeshop/testkube/pkg/tcl/checktcl"
 )
 
 func ExitOnError(title string, err error) {
@@ -86,7 +90,6 @@ func HandleCancelSignal(ctx context.Context) func() error {
 func MustGetConfig() *config.Config {
 	cfg, err := config.Get()
 	ExitOnError("error getting application config", err)
-	cfg.CleanLegacyVars()
 	return cfg
 }
 
@@ -282,29 +285,33 @@ func ReadDefaultExecutors(cfg *config.Config) (executors []testkube.ExecutorDeta
 
 func ReadProContext(ctx context.Context, cfg *config.Config, grpcClient cloud.TestKubeCloudAPIClient) config.ProContext {
 	proContext := config.ProContext{
-		APIKey:                                  cfg.TestkubeProAPIKey,
-		URL:                                     cfg.TestkubeProURL,
-		TLSInsecure:                             cfg.TestkubeProTLSInsecure,
-		WorkerCount:                             cfg.TestkubeProWorkerCount,
-		LogStreamWorkerCount:                    cfg.TestkubeProLogStreamWorkerCount,
-		WorkflowNotificationsWorkerCount:        cfg.TestkubeProWorkflowNotificationsWorkerCount,
-		WorkflowServiceNotificationsWorkerCount: cfg.TestkubeProWorkflowServiceNotificationsWorkerCount,
-		WorkflowParallelStepNotificationsWorkerCount: cfg.TestkubeProWorkflowParallelStepNotificationsWorkerCount,
-		SkipVerify:        cfg.TestkubeProSkipVerify,
-		EnvID:             cfg.TestkubeProEnvID,
-		OrgID:             cfg.TestkubeProOrgID,
-		Migrate:           cfg.TestkubeProMigrate,
-		ConnectionTimeout: cfg.TestkubeProConnectionTimeout,
-		DashboardURI:      cfg.TestkubeDashboardURI,
+		APIKey:                              cfg.ControlPlaneConfig.TestkubeProAPIKey,
+		URL:                                 cfg.ControlPlaneConfig.TestkubeProURL,
+		TLSInsecure:                         cfg.ControlPlaneConfig.TestkubeProTLSInsecure,
+		SkipVerify:                          cfg.ControlPlaneConfig.TestkubeProSkipVerify,
+		AgentID:                             cfg.ControlPlaneConfig.TestkubeProAgentID,
+		EnvID:                               cfg.ControlPlaneConfig.TestkubeProEnvID,
+		OrgID:                               cfg.ControlPlaneConfig.TestkubeProOrgID,
+		ConnectionTimeout:                   cfg.ControlPlaneConfig.TestkubeProConnectionTimeout,
+		WorkerCount:                         cfg.TestkubeProWorkerCount,
+		LogStreamWorkerCount:                cfg.TestkubeProLogStreamWorkerCount,
+		Migrate:                             cfg.TestkubeProMigrate,
+		DashboardURI:                        cfg.TestkubeDashboardURI,
+		NewArchitecture:                     grpcClient == nil,
+		CloudStorage:                        grpcClient == nil,
+		CloudStorageSupportedInControlPlane: grpcClient == nil,
 	}
 
 	if cfg.TestkubeProAPIKey == "" || grpcClient == nil {
 		return proContext
 	}
 
 	ctx, cancel := context.WithTimeout(ctx, time.Second*3)
-	md := metadata.Pairs("api-key", cfg.TestkubeProAPIKey)
-	ctx = metadata.NewOutgoingContext(ctx, md)
+	ctx = metadata.NewOutgoingContext(ctx, metadata.New(map[string]string{
+		"api-key":         cfg.TestkubeProAPIKey,
+		"organization-id": cfg.TestkubeProOrgID,
+		"agent-id":        cfg.TestkubeProAgentID,
+	}))
 	defer cancel()
 	foundProContext, err := grpcClient.GetProContext(ctx, &emptypb.Empty{})
 	if err != nil {
@@ -316,10 +323,45 @@ func ReadProContext(ctx context.Context, cfg *config.Config, grpcClient cloud.Te
 		proContext.EnvID = foundProContext.EnvId
 	}
 
+	if proContext.AgentID == "" && strings.HasPrefix(proContext.APIKey, "tkcagnt_") {
+		proContext.AgentID = strings.Replace(foundProContext.EnvId, "tkcenv_", "tkcroot_", 1)
+	}
+
 	if proContext.OrgID == "" {
 		proContext.OrgID = foundProContext.OrgId
 	}
 
+	if cfg.FeatureNewArchitecture && capabilities.Enabled(foundProContext.Capabilities, capabilities.CapabilityNewArchitecture) {
+		proContext.NewArchitecture = true
+	}
+
+	if capabilities.Enabled(foundProContext.Capabilities, capabilities.CapabilityCloudStorage) {
+		proContext.CloudStorageSupportedInControlPlane = true
+		if cfg.FeatureCloudStorage {
+			proContext.CloudStorage = true
+		}
+	}
+
+	if string(foundProContext.Mode) != "" {
+		proContext.IsTrial = foundProContext.Trial
+		proContext.Mode = config.ProContextMode(foundProContext.Mode.String())
+		proContext.Status = config.ProContextStatus(foundProContext.Status.String())
+	} else {
+		req := checktcl.GetOrganizationPlanRequest{}
+		response, err := executor.NewCloudGRPCExecutor(grpcClient, proContext.APIKey).
+			Execute(ctx, cloudconfig.CmdConfigGetOrganizationPlan, req)
+		if err != nil {
+			return proContext
+		}
+		var commandResponse checktcl.GetOrganizationPlanResponse
+		if err := json.Unmarshal(response, &commandResponse); err != nil {
+			return proContext
+		}
+		proContext.IsTrial = commandResponse.IsTrial
+		proContext.Mode = config.ProContextMode(commandResponse.TestkubeMode)
+		proContext.Status = config.ProContextStatus(commandResponse.PlanStatus)
+	}
+
 	return proContext
 }
 
@@ -367,7 +409,7 @@ func MustCreateNATSConnection(cfg *config.Config) *nats.EncodedConn {
 
 // Components
 
-func CreateImageInspector(cfg *config.Config, configMapClient configmap.Interface, secretClient secret.Interface) imageinspector.Inspector {
+func CreateImageInspector(cfg *config.ImageInspectorConfig, configMapClient configmap.Interface, secretClient secret.Interface) imageinspector.Inspector {
 	inspectorStorages := []imageinspector.Storage{imageinspector.NewMemoryStorage()}
 	if cfg.EnableImageDataPersistentCache {
 		configmapStorage := imageinspector.NewConfigMapStorage(configMapClient, cfg.ImageDataPersistentCacheKey, true)

cmd/api-server/commons/deprecated.go

@@ -4,7 +4,6 @@ import (
 	"context"
 
 	"go.mongodb.org/mongo-driver/mongo"
-	"google.golang.org/grpc"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	executorsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/executors/v1"
@@ -109,10 +108,10 @@ func (d *deprecatedRepositories) TestSuiteResults() testresult.Repository {
 	return d.testSuiteResults
 }
 
-func CreateDeprecatedRepositoriesForCloud(grpcClient cloud.TestKubeCloudAPIClient, grpcConn *grpc.ClientConn, apiKey string) DeprecatedRepositories {
+func CreateDeprecatedRepositoriesForCloud(grpcClient cloud.TestKubeCloudAPIClient, apiKey string) DeprecatedRepositories {
 	return &deprecatedRepositories{
-		testResults:      cloudresult.NewCloudResultRepository(grpcClient, grpcConn, apiKey),
-		testSuiteResults: cloudtestresult.NewCloudRepository(grpcClient, grpcConn, apiKey),
+		testResults:      cloudresult.NewCloudResultRepository(grpcClient, apiKey),
+		testSuiteResults: cloudtestresult.NewCloudRepository(grpcClient, apiKey),
 	}
 }