Add backend selection for agent

agent/runner.go

@@ -28,7 +28,7 @@ import (
 	"google.golang.org/grpc/metadata"
 
 	"github.com/woodpecker-ci/woodpecker/pipeline"
-	"github.com/woodpecker-ci/woodpecker/pipeline/backend"
+	backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
 	"github.com/woodpecker-ci/woodpecker/pipeline/multipart"
 	"github.com/woodpecker-ci/woodpecker/pipeline/rpc"
 )

cli/exec/exec.go

@@ -15,8 +15,8 @@ import (
 
 	"github.com/woodpecker-ci/woodpecker/cli/common"
 	"github.com/woodpecker-ci/woodpecker/pipeline"
-	"github.com/woodpecker-ci/woodpecker/pipeline/backend"
 	"github.com/woodpecker-ci/woodpecker/pipeline/backend/docker"
+	backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
 	"github.com/woodpecker-ci/woodpecker/pipeline/frontend"
 	"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
 	"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/compiler"
@@ -158,8 +158,8 @@ func execWithAxis(c *cli.Context, axis matrix.Axis) error {
 		compiler.WithSecret(secrets...),
 		compiler.WithEnviron(droneEnv),
 	).Compile(conf)
-	engine, err := docker.NewEnv()
-	if err != nil {
+	engine := docker.New()
+	if err = engine.Load(); err != nil {
 		return err
 	}
 

cmd/agent/agent.go

@@ -31,7 +31,7 @@ import (
 	"google.golang.org/grpc/metadata"
 
 	"github.com/woodpecker-ci/woodpecker/agent"
-	"github.com/woodpecker-ci/woodpecker/pipeline/backend/docker"
+	"github.com/woodpecker-ci/woodpecker/pipeline/backend"
 	"github.com/woodpecker-ci/woodpecker/pipeline/rpc"
 )
 
@@ -138,13 +138,22 @@ func loop(c *cli.Context) error {
 					return
 				}
 
-				// new docker engine
-				engine, err := docker.NewEnv()
+				// new engine
+				engine, err := backend.FindEngine(c.String("backend-engine"))
 				if err != nil {
-					log.Error().Err(err).Msg("cannot create docker client")
+					log.Error().Err(err).Msgf("cannot find backend engine '%s'", c.String("backend-engine"))
 					return
 				}
 
+				// load enginge (e.g. init api client)
+				err = engine.Load()
+				if err != nil {
+					log.Error().Err(err).Msg("cannot load backend engine")
+					return
+				}
+
+				log.Debug().Msgf("loaded %s backend engine", engine.Name())
+
 				r := agent.NewRunner(client, filter, hostname, counter, &engine)
 				if err := r.Run(ctx); err != nil {
 					log.Error().Err(err).Msg("pipeline done with error")

cmd/agent/flags.go

@@ -110,4 +110,10 @@ var flags = []cli.Flag{
 		Usage:   "should the grpc server certificate be verified, only valid when WOODPECKER_GRPC_SECURE is true",
 		Value:   true,
 	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND"},
+		Name:    "backend-engine",
+		Usage:   "backend engine to run pipelines on",
+		Value:   "auto-detect",
+	},
 }

pipeline/backend/backend.go

@@ -1,29 +1,44 @@
 package backend
 
 import (
-	"context"
-	"io"
+	"fmt"
+
+	"github.com/woodpecker-ci/woodpecker/pipeline/backend/docker"
+	"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
+)
+
+var (
+	engines map[string]types.Engine
 )
 
-// Engine defines a container orchestration backend and is used
-// to create and manage container resources.
-type Engine interface {
-	// Setup the pipeline environment.
-	Setup(context.Context, *Config) error
+func init() {
+	engines = make(map[string]types.Engine)
 
-	// Exec start the pipeline step.
-	Exec(context.Context, *Step) error
+	// TODO: disabled for now as kubernetes backend has not been implemented yet
+	// kubernetes
+	// engine = kubernetes.New("", "", "")
+	// engines[engine.Name()] = engine
+
+	// docker
+	engine := docker.New()
+	engines[engine.Name()] = engine
+}
 
-	// Kill the pipeline step.
-	Kill(context.Context, *Step) error
+func FindEngine(engineName string) (types.Engine, error) {
+	if engineName == "auto-detect" {
+		for _, engine := range engines {
+			if engine.IsAvivable() {
+				return engine, nil
+			}
+		}
 
-	// Wait for the pipeline step to complete and returns
-	// the completion results.
-	Wait(context.Context, *Step) (*State, error)
+		return nil, fmt.Errorf("Can't detect an avivable backend engine")
+	}
 
-	// Tail the pipeline step logs.
-	Tail(context.Context, *Step) (io.ReadCloser, error)
+	engine, ok := engines[engineName]
+	if !ok {
+		return nil, fmt.Errorf("Backend engine '%s' not found", engineName)
+	}
 
-	// Destroy the pipeline environment.
-	Destroy(context.Context, *Config) error
+	return engine, nil
 }

pipeline/backend/docker/convert.go

@@ -8,11 +8,11 @@ import (
 
 	"github.com/docker/docker/api/types/container"
 
-	"github.com/woodpecker-ci/woodpecker/pipeline/backend"
+	"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
 )
 
 // returns a container configuration.
-func toConfig(proc *backend.Step) *container.Config {
+func toConfig(proc *types.Step) *container.Config {
 	config := &container.Config{
 		Image:        proc.Image,
 		Labels:       proc.Labels,
@@ -36,7 +36,7 @@ func toConfig(proc *backend.Step) *container.Config {
 }
 
 // returns a container host configuration.
-func toHostConfig(proc *backend.Step) *container.HostConfig {
+func toHostConfig(proc *types.Step) *container.HostConfig {
 	config := &container.HostConfig{
 		Resources: container.Resources{
 			CPUQuota:   proc.CPUQuota,
@@ -146,7 +146,7 @@ func toDev(paths []string) []container.DeviceMapping {
 
 // helper function that serializes the auth configuration as JSON
 // base64 payload.
-func encodeAuthToBase64(authConfig backend.Auth) (string, error) {
+func encodeAuthToBase64(authConfig types.Auth) (string, error) {
 	buf, err := json.Marshal(authConfig)
 	if err != nil {
 		return "", err

pipeline/backend/docker/docker.go

@@ -14,28 +14,38 @@ import (
 	"github.com/moby/term"
 	"github.com/rs/zerolog/log"
 
-	"github.com/woodpecker-ci/woodpecker/pipeline/backend"
+	backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
 )
 
 type engine struct {
 	client client.APIClient
 }
 
-// New returns a new Docker Engine using the given client.
-func New(cli client.APIClient) backend.Engine {
+// New returns a new Docker Engine.
+func New() backend.Engine {
 	return &engine{
-		client: cli,
+		client: nil,
 	}
 }
 
-// NewEnv returns a new Docker Engine using the client connection
-// environment variables.
-func NewEnv() (backend.Engine, error) {
+func (e *engine) Name() string {
+	return "docker"
+}
+
+func (e *engine) IsAvivable() bool {
+	_, err := os.Stat("/.dockerenv")
+	return os.IsNotExist(err)
+}
+
+// Load new client for Docker Engine using environment variables.
+func (e *engine) Load() error {
 	cli, err := client.NewClientWithOpts(client.FromEnv)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	return New(cli), nil
+	e.client = cli
+
+	return nil
 }
 
 func (e *engine) Setup(_ context.Context, conf *backend.Config) error {

pipeline/backend/kubernetes/kubernetes.go

@@ -3,8 +3,9 @@ package kubernetes
 import (
 	"context"
 	"io"
+	"os"
 
-	"github.com/woodpecker-ci/woodpecker/pipeline/backend"
+	"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
 )
 
 type engine struct {
@@ -14,48 +15,61 @@ type engine struct {
 }
 
 // New returns a new Kubernetes Engine.
-func New(namespace, endpoint, token string) backend.Engine {
+func New(namespace, endpoint, token string) types.Engine {
 	return &engine{
 		namespace: namespace,
 		endpoint:  endpoint,
 		token:     token,
 	}
 }
 
+func (e *engine) Name() string {
+	return "kubernetes"
+}
+
+func (e *engine) IsAvivable() bool {
+	host := os.Getenv("KUBERNETES_SERVICE_HOST")
+	return len(host) > 0
+}
+
+func (e *engine) Load() error {
+	return nil
+}
+
 // Setup the pipeline environment.
-func (e *engine) Setup(context.Context, *backend.Config) error {
+func (e *engine) Setup(context.Context, *types.Config) error {
 	// POST /api/v1/namespaces
 	return nil
 }
 
 // Start the pipeline step.
-func (e *engine) Exec(context.Context, *backend.Step) error {
+func (e *engine) Exec(context.Context, *types.Step) error {
 	// POST /api/v1/namespaces/{namespace}/pods
 	return nil
 }
 
 // DEPRECATED
 // Kill the pipeline step.
-func (e *engine) Kill(context.Context, *backend.Step) error {
+func (e *engine) Kill(context.Context, *types.Step) error {
 	return nil
 }
 
 // Wait for the pipeline step to complete and returns
 // the completion results.
-func (e *engine) Wait(context.Context, *backend.Step) (*backend.State, error) {
+func (e *engine) Wait(context.Context, *types.Step) (*types.State, error) {
 	// GET /api/v1/watch/namespaces/{namespace}/pods
 	// GET /api/v1/watch/namespaces/{namespace}/pods/{name}
 	return nil, nil
 }
 
 // Tail the pipeline step logs.
-func (e *engine) Tail(context.Context, *backend.Step) (io.ReadCloser, error) {
+func (e *engine) Tail(context.Context, *types.Step) (io.ReadCloser, error) {
 	// GET /api/v1/namespaces/{namespace}/pods/{name}/log
 	return nil, nil
 }
 
 // Destroy the pipeline environment.
-func (e *engine) Destroy(context.Context, *backend.Config) error {
+func (e *engine) Destroy(context.Context, *types.Config) error {
 	// DELETE /api/v1/namespaces/{name}
 	return nil
 }