Parse backend options in backend

go.mod

@@ -32,6 +32,7 @@ require (
 	github.com/kinbiko/jsonassert v1.1.1
 	github.com/lib/pq v1.10.9
 	github.com/mattn/go-sqlite3 v1.14.19
+	github.com/mitchellh/mapstructure v1.4.2
 	github.com/moby/moby v24.0.7+incompatible
 	github.com/moby/term v0.5.0
 	github.com/muesli/termenv v0.15.2

go.sum

@@ -308,6 +308,8 @@ github.com/mholt/acmez v1.2.0 h1:1hhLxSgY5FvH5HCnGUuwbKY2VQVo8IU7rxXKSnZ7F30=
 github.com/mholt/acmez v1.2.0/go.mod h1:VT9YwH1xgNX1kmYY89gY8xPJC84BFAisjo8Egigt4kE=
 github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM=
 github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk=
+github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo=
+github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/moby/moby v24.0.7+incompatible h1:RrVT5IXBn85mRtFKP+gFwVLCcnNPZIgN3NVRJG9Le+4=
 github.com/moby/moby v24.0.7+incompatible/go.mod h1:fDXVQ6+S340veQPv35CzDahGBmHsiclFwfEygB/TWMc=
 github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
@@ -417,8 +419,6 @@ github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65E
 github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
 github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
 github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
-github.com/xanzy/go-gitlab v0.95.2 h1:4p0IirHqEp5f0baK/aQqr4TR57IsD+8e4fuyAA1yi88=
-github.com/xanzy/go-gitlab v0.95.2/go.mod h1:ETg8tcj4OhrB84UEgeE8dSuV/0h4BBL1uOV/qK0vlyI=
 github.com/xanzy/go-gitlab v0.96.0 h1:LGkZ+wSNMRtHIBaYE4Hq3dZVjprwHv3Y1+rhKU3WETs=
 github.com/xanzy/go-gitlab v0.96.0/go.mod h1:ETg8tcj4OhrB84UEgeE8dSuV/0h4BBL1uOV/qK0vlyI=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -622,16 +622,10 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
 gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A=
-k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA=
 k8s.io/api v0.29.1 h1:DAjwWX/9YT7NQD4INu49ROJuZAAAP/Ijki48GUPzxqw=
 k8s.io/api v0.29.1/go.mod h1:7Kl10vBRUXhnQQI8YR/R327zXC8eJ7887/+Ybta+RoQ=
-k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o=
-k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis=
 k8s.io/apimachinery v0.29.1 h1:KY4/E6km/wLBguvCZv8cKTeOwwOBqFNjwJIdMkMbbRc=
 k8s.io/apimachinery v0.29.1/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU=
-k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8=
-k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38=
 k8s.io/client-go v0.29.1 h1:19B/+2NGEwnFLzt0uB5kNJnfTsbV8w6TgQRz9l7ti7A=
 k8s.io/client-go v0.29.1/go.mod h1:TDG/psL9hdet0TI9mGyHJSgRkW3H9JZk2dNEUS7bRks=
 k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=

pipeline/backend/docker/convert_test.go

@@ -151,7 +151,7 @@ func TestToConfigFull(t *testing.T) {
 		OnFailure:    true,
 		OnSuccess:    true,
 		Failure:      "fail",
-		AuthConfig:   backend.Auth{Username: "user", Password: "123456", Email: "user@example.com"},
+		AuthConfig:   backend.Auth{Username: "user", Password: "123456"},
 		NetworkMode:  "bridge",
 		Ports:        []backend.Port{{Number: 21}, {Number: 22}},
 	})

pipeline/backend/kubernetes/backend_options.go

@@ -0,0 +1,74 @@
+package kubernetes
+
+import (
+	"github.com/mitchellh/mapstructure"
+
+	backend "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
+)
+
+// BackendOptions defines all the advanced options for the kubernetes backend
+type BackendOptions struct {
+	Resources          Resources         `mapstructure:"resources"`
+	ServiceAccountName string            `mapstructure:"serviceAccountName"`
+	NodeSelector       map[string]string `mapstructure:"nodeSelector"`
+	Tolerations        []Toleration      `mapstructure:"tolerations"`
+	SecurityContext    *SecurityContext  `mapstructure:"securityContext"`
+}
+
+// Resources defines two maps for kubernetes resource definitions
+type Resources struct {
+	Requests map[string]string `mapstructure:"requests"`
+	Limits   map[string]string `mapstructure:"limits"`
+}
+
+// Toleration defines Kubernetes toleration
+type Toleration struct {
+	Key               string             `mapstructure:"key"`
+	Operator          TolerationOperator `mapstructure:"operator"`
+	Value             string             `mapstructure:"value"`
+	Effect            TaintEffect        `mapstructure:"effect"`
+	TolerationSeconds *int64             `mapstructure:"tolerationSeconds"`
+}
+
+type TaintEffect string
+
+const (
+	TaintEffectNoSchedule       TaintEffect = "NoSchedule"
+	TaintEffectPreferNoSchedule TaintEffect = "PreferNoSchedule"
+	TaintEffectNoExecute        TaintEffect = "NoExecute"
+)
+
+type TolerationOperator string
+
+const (
+	TolerationOpExists TolerationOperator = "Exists"
+	TolerationOpEqual  TolerationOperator = "Equal"
+)
+
+type SecurityContext struct {
+	Privileged      *bool       `mapstructure:"privileged"`
+	RunAsNonRoot    *bool       `mapstructure:"runAsNonRoot"`
+	RunAsUser       *int64      `mapstructure:"runAsUser"`
+	RunAsGroup      *int64      `mapstructure:"runAsGroup"`
+	FSGroup         *int64      `mapstructure:"fsGroup"`
+	SeccompProfile  *SecProfile `mapstructure:"seccompProfile"`
+	ApparmorProfile *SecProfile `mapstructure:"apparmorProfile"`
+}
+
+type SecProfile struct {
+	Type             SecProfileType `mapstructure:"type"`
+	LocalhostProfile string         `mapstructure:"localhostProfile"`
+}
+
+type SecProfileType string
+
+const (
+	SecProfileTypeRuntimeDefault SecProfileType = "RuntimeDefault"
+	SecProfileTypeLocalhost      SecProfileType = "Localhost"
+)
+
+func parseBackendOptions(step *backend.Step) (BackendOptions, error) {
+	var result BackendOptions
+	err := mapstructure.Decode(step.BackendOptions[EngineName], &result)
+	return result, err
+}

pipeline/backend/kubernetes/kubernetes.go

@@ -168,7 +168,7 @@ func (e *kube) getConfig() *config {
 	return &c
 }
 
-// Setup the pipeline environment.
+// SetupWorkflow sets up the pipeline environment.
 func (e *kube) SetupWorkflow(ctx context.Context, conf *types.Config, taskUUID string) error {
 	log.Trace().Str("taskUUID", taskUUID).Msgf("Setting up Kubernetes primitives")
 
@@ -179,7 +179,7 @@ func (e *kube) SetupWorkflow(ctx context.Context, conf *types.Config, taskUUID s
 		}
 	}
 
-	extraHosts := []types.HostAlias{}
+	var extraHosts []types.HostAlias
 	for _, stage := range conf.Stages {
 		for _, step := range stage.Steps {
 			if step.Type == types.StepTypeService {
@@ -202,7 +202,7 @@ func (e *kube) SetupWorkflow(ctx context.Context, conf *types.Config, taskUUID s
 	return nil
 }
 
-// Start the pipeline step.
+// StartStep starts the pipeline step.
 func (e *kube) StartStep(ctx context.Context, step *types.Step, taskUUID string) error {
 	if step.Type == types.StepTypeService {
 		// a service should be started by SetupWorkflow so we can ignore it
@@ -214,7 +214,7 @@ func (e *kube) StartStep(ctx context.Context, step *types.Step, taskUUID string)
 	return err
 }
 
-// Wait for the pipeline step to complete and returns
+// WaitStep waits for the pipeline step to complete and returns
 // the completion results.
 func (e *kube) WaitStep(ctx context.Context, step *types.Step, taskUUID string) (*types.State, error) {
 	podName, err := stepToPodName(step)
@@ -280,7 +280,7 @@ func (e *kube) WaitStep(ctx context.Context, step *types.Step, taskUUID string)
 	return bs, nil
 }
 
-// Tail the pipeline step logs.
+// TailStep tails the pipeline step logs.
 func (e *kube) TailStep(ctx context.Context, step *types.Step, taskUUID string) (io.ReadCloser, error) {
 	podName, err := stepToPodName(step)
 	if err != nil {
@@ -363,7 +363,7 @@ func (e *kube) DestroyStep(ctx context.Context, step *types.Step, taskUUID strin
 	return err
 }
 
-// Destroy the pipeline environment.
+// DestroyWorkflow destroys the pipeline environment.
 func (e *kube) DestroyWorkflow(ctx context.Context, conf *types.Config, taskUUID string) error {
 	log.Trace().Str("taskUUID", taskUUID).Msg("deleting Kubernetes primitives")
 

pipeline/backend/kubernetes/pod.go

@@ -36,14 +36,18 @@ const (
 )
 
 func mkPod(step *types.Step, config *config, podName, goos string) (*v1.Pod, error) {
-	meta := podMeta(step, config, podName)
+	options, err := parseBackendOptions(step)
+	if err != nil {
+		log.Error().Err(err).Msg("could not parse backend options")
+	}
+	meta := podMeta(step, config, options, podName)
 
-	spec, err := podSpec(step, config)
+	spec, err := podSpec(step, config, options)
 	if err != nil {
 		return nil, err
 	}
 
-	container, err := podContainer(step, podName, goos)
+	container, err := podContainer(step, podName, goos, options)
 	if err != nil {
 		return nil, err
 	}
@@ -68,7 +72,7 @@ func podName(step *types.Step) (string, error) {
 	return dnsName(podPrefix + step.UUID)
 }
 
-func podMeta(step *types.Step, config *config, podName string) metav1.ObjectMeta {
+func podMeta(step *types.Step, config *config, options BackendOptions, podName string) metav1.ObjectMeta {
 	meta := metav1.ObjectMeta{
 		Name:      podName,
 		Namespace: config.Namespace,
@@ -85,7 +89,7 @@ func podMeta(step *types.Step, config *config, podName string) metav1.ObjectMeta
 		meta.Annotations = make(map[string]string)
 	}
 
-	securityContext := step.BackendOptions.Kubernetes.SecurityContext
+	securityContext := options.SecurityContext
 	if securityContext != nil {
 		key, value := apparmorAnnotation(podName, securityContext.ApparmorProfile)
 		if key != nil && value != nil {
@@ -96,16 +100,16 @@ func podMeta(step *types.Step, config *config, podName string) metav1.ObjectMeta
 	return meta
 }
 
-func podSpec(step *types.Step, config *config) (v1.PodSpec, error) {
+func podSpec(step *types.Step, config *config, options BackendOptions) (v1.PodSpec, error) {
 	var err error
 	spec := v1.PodSpec{
 		RestartPolicy:      v1.RestartPolicyNever,
-		ServiceAccountName: step.BackendOptions.Kubernetes.ServiceAccountName,
+		ServiceAccountName: options.ServiceAccountName,
 		ImagePullSecrets:   imagePullSecretsReferences(config.ImagePullSecretNames),
 		HostAliases:        hostAliases(step.ExtraHosts),
-		NodeSelector:       nodeSelector(step.BackendOptions.Kubernetes.NodeSelector, step.Environment["CI_SYSTEM_PLATFORM"]),
-		Tolerations:        tolerations(step.BackendOptions.Kubernetes.Tolerations),
-		SecurityContext:    podSecurityContext(step.BackendOptions.Kubernetes.SecurityContext, config.SecurityContext),
+		NodeSelector:       nodeSelector(options.NodeSelector, step.Environment["CI_SYSTEM_PLATFORM"]),
+		Tolerations:        tolerations(options.Tolerations),
+		SecurityContext:    podSecurityContext(options.SecurityContext, config.SecurityContext),
 	}
 	spec.Volumes, err = volumes(step.Volumes)
 	if err != nil {
@@ -115,7 +119,7 @@ func podSpec(step *types.Step, config *config) (v1.PodSpec, error) {
 	return spec, nil
 }
 
-func podContainer(step *types.Step, podName, goos string) (v1.Container, error) {
+func podContainer(step *types.Step, podName, goos string, options BackendOptions) (v1.Container, error) {
 	var err error
 	container := v1.Container{
 		Name:       podName,
@@ -139,9 +143,9 @@ func podContainer(step *types.Step, podName, goos string) (v1.Container, error)
 
 	container.Env = mapToEnvVars(step.Environment)
 	container.Ports = containerPorts(step.Ports)
-	container.SecurityContext = containerSecurityContext(step.BackendOptions.Kubernetes.SecurityContext, step.Privileged)
+	container.SecurityContext = containerSecurityContext(options.SecurityContext, step.Privileged)
 
-	container.Resources, err = resourceRequirements(step.BackendOptions.Kubernetes.Resources)
+	container.Resources, err = resourceRequirements(options.Resources)
 	if err != nil {
 		return container, err
 	}
@@ -251,7 +255,7 @@ func imagePullSecretsReference(imagePullSecretName string) v1.LocalObjectReferen
 	}
 }
 
-func resourceRequirements(resources types.Resources) (v1.ResourceRequirements, error) {
+func resourceRequirements(resources Resources) (v1.ResourceRequirements, error) {
 	var err error
 	requirements := v1.ResourceRequirements{}
 
@@ -298,7 +302,7 @@ func nodeSelector(backendNodeSelector map[string]string, platform string) map[st
 	return nodeSelector
 }
 
-func tolerations(backendTolerations []types.Toleration) []v1.Toleration {
+func tolerations(backendTolerations []Toleration) []v1.Toleration {
 	var tolerations []v1.Toleration
 
 	if len(backendTolerations) > 0 {
@@ -312,7 +316,7 @@ func tolerations(backendTolerations []types.Toleration) []v1.Toleration {
 	return tolerations
 }
 
-func toleration(backendToleration types.Toleration) v1.Toleration {
+func toleration(backendToleration Toleration) v1.Toleration {
 	return v1.Toleration{
 		Key:               backendToleration.Key,
 		Operator:          v1.TolerationOperator(backendToleration.Operator),
@@ -322,7 +326,7 @@ func toleration(backendToleration types.Toleration) v1.Toleration {
 	}
 }
 
-func podSecurityContext(sc *types.SecurityContext, secCtxConf SecurityContextConfig) *v1.PodSecurityContext {
+func podSecurityContext(sc *SecurityContext, secCtxConf SecurityContextConfig) *v1.PodSecurityContext {
 	var (
 		nonRoot *bool
 		user    *int64
@@ -364,7 +368,7 @@ func podSecurityContext(sc *types.SecurityContext, secCtxConf SecurityContextCon
 	return securityContext
 }
 
-func seccompProfile(scp *types.SecProfile) *v1.SeccompProfile {
+func seccompProfile(scp *SecProfile) *v1.SeccompProfile {
 	if scp == nil || len(scp.Type) == 0 {
 		return nil
 	}
@@ -380,7 +384,7 @@ func seccompProfile(scp *types.SecProfile) *v1.SeccompProfile {
 	return seccompProfile
 }
 
-func containerSecurityContext(sc *types.SecurityContext, stepPrivileged bool) *v1.SecurityContext {
+func containerSecurityContext(sc *SecurityContext, stepPrivileged bool) *v1.SecurityContext {
 	var privileged *bool
 
 	if sc != nil && sc.Privileged != nil && *sc.Privileged {
@@ -400,7 +404,7 @@ func containerSecurityContext(sc *types.SecurityContext, stepPrivileged bool) *v
 	return securityContext
 }
 
-func apparmorAnnotation(containerName string, scp *types.SecProfile) (*string, *string) {
+func apparmorAnnotation(containerName string, scp *SecProfile) (*string, *string) {
 	if scp == nil {
 		return nil, nil
 	}
@@ -411,12 +415,12 @@ func apparmorAnnotation(containerName string, scp *types.SecProfile) (*string, *
 		profilePath string
 	)
 
-	if scp.Type == types.SecProfileTypeRuntimeDefault {
+	if scp.Type == SecProfileTypeRuntimeDefault {
 		profileType = "runtime"
 		profilePath = "default"
 	}
 
-	if scp.Type == types.SecProfileTypeLocalhost {
+	if scp.Type == SecProfileTypeLocalhost {
 		profileType = "localhost"
 		profilePath = scp.LocalhostProfile
 	}