Move constrain to only have a single command in backend to run to ded…

…icated backends (#1032)

at the moment we compile a script that we can pipe in as single command
this is because of the constrains the docker backend gives us.

so we move it into the docker backend and eventually get rid of it altogether

pipeline/backend/common/script.go

@@ -0,0 +1,43 @@
+// Copyright 2022 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import "runtime"
+
+func GenerateContainerConf(commands []string) (env map[string]string, entry, cmd []string) {
+	env = make(map[string]string)
+	if runtime.GOOS == "windows" {
+		env["CI_SCRIPT"] = generateScriptWindows(commands)
+		env["HOME"] = "c:\\root"
+		env["SHELL"] = "powershell.exe"
+		entry = []string{"powershell", "-noprofile", "-noninteractive", "-command"}
+		cmd = []string{"[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($Env:CI_SCRIPT)) | iex"}
+	} else {
+		env["CI_SCRIPT"] = generateScriptPosix(commands)
+		env["HOME"] = "/root"
+		env["SHELL"] = "/bin/sh"
+		entry = []string{"/bin/sh", "-c"}
+		cmd = []string{"echo $CI_SCRIPT | base64 -d | /bin/sh -e"}
+	}
+
+	return env, entry, cmd
+}
+
+func GenerateScript(commands []string) string {
+	if runtime.GOOS == "windows" {
+		return generateScriptWindows(commands)
+	}
+	return generateScriptPosix(commands)
+}

pipeline/frontend/yaml/compiler/script_posix.go → pipeline/backend/common/script_posix.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package compiler
+package common
 
 import (
 	"bytes"

pipeline/frontend/yaml/compiler/script_posix_test.go → pipeline/backend/common/script_posix_test.go

@@ -1,4 +1,18 @@
-package compiler
+// Copyright 2022 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
 
 import (
 	"encoding/base64"

pipeline/frontend/yaml/compiler/script_win.go → pipeline/backend/common/script_win.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package compiler
+package common
 
 import (
 	"bytes"

pipeline/backend/docker/convert.go

@@ -1,3 +1,17 @@
+// Copyright 2022 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package docker
 
 import (
@@ -8,6 +22,7 @@ import (
 
 	"github.com/docker/docker/api/types/container"
 
+	"github.com/woodpecker-ci/woodpecker/pipeline/backend/common"
 	"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
 )
 
@@ -20,15 +35,19 @@ func toConfig(step *types.Step) *container.Config {
 		AttachStdout: true,
 		AttachStderr: true,
 	}
+
+	if len(step.Commands) != 0 {
+		env, entry, cmd := common.GenerateContainerConf(step.Commands)
+		for k, v := range env {
+			step.Environment[k] = v
+		}
+		config.Entrypoint = entry
+		config.Cmd = cmd
+	}
+
 	if len(step.Environment) != 0 {
 		config.Env = toEnv(step.Environment)
 	}
-	if len(step.Command) != 0 {
-		config.Cmd = step.Command
-	}
-	if len(step.Entrypoint) != 0 {
-		config.Entrypoint = step.Entrypoint
-	}
 	if len(step.Volumes) != 0 {
 		config.Volumes = toVol(step.Volumes)
 	}

pipeline/backend/docker/convert_test.go

@@ -1,3 +1,17 @@
+// Copyright 2022 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package docker
 
 import (

pipeline/backend/docker/docker.go

@@ -1,3 +1,17 @@
+// Copyright 2022 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package docker
 
 import (

pipeline/backend/kubernetes/pod.go

@@ -3,15 +3,21 @@ package kubernetes
 import (
 	"strings"
 
+	"github.com/woodpecker-ci/woodpecker/pipeline/backend/common"
 	"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func Pod(namespace string, step *types.Step) *v1.Pod {
-	var vols []v1.Volume
-	var volMounts []v1.VolumeMount
+	var (
+		vols       []v1.Volume
+		volMounts  []v1.VolumeMount
+		entrypoint []string
+		args       []string
+	)
+
 	if step.WorkingDir != "" {
 		for _, vol := range step.Volumes {
 			vols = append(vols, v1.Volume{
@@ -36,13 +42,13 @@ func Pod(namespace string, step *types.Step) *v1.Pod {
 		pullPolicy = v1.PullAlways
 	}
 
-	command := step.Entrypoint
-	args := step.Command
-	envs := mapToEnvVars(step.Environment)
-
-	if _, hasScript := step.Environment["CI_SCRIPT"]; !strings.HasSuffix(step.Name, "_clone") && hasScript {
-		command = []string{"/bin/sh", "-c"}
-		args = []string{"echo $CI_SCRIPT | base64 -d | /bin/sh -e"}
+	if len(step.Commands) != 0 {
+		scriptEnv, entry, cmds := common.GenerateContainerConf(step.Commands)
+		for k, v := range scriptEnv {
+			step.Environment[k] = v
+		}
+		entrypoint = entry
+		args = cmds
 	}
 
 	hostAliases := []v1.HostAlias{}
@@ -97,10 +103,10 @@ func Pod(namespace string, step *types.Step) *v1.Pod {
 				Name:            podName(step),
 				Image:           step.Image,
 				ImagePullPolicy: pullPolicy,
-				Command:         command,
+				Command:         entrypoint,
 				Args:            args,
 				WorkingDir:      step.WorkingDir,
-				Env:             envs,
+				Env:             mapToEnvVars(step.Environment),
 				VolumeMounts:    volMounts,
 				Resources:       resources,
 				SecurityContext: &v1.SecurityContext{

pipeline/backend/local/local.go

@@ -16,12 +16,12 @@ package local
 
 import (
 	"context"
-	"encoding/base64"
 	"io"
 	"os"
 	"os/exec"
 	"strings"
 
+	"github.com/woodpecker-ci/woodpecker/pipeline/backend/common"
 	"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
 	"github.com/woodpecker-ci/woodpecker/shared/constant"
 )
@@ -62,32 +62,32 @@ func (e *local) Setup(ctx context.Context, config *types.Config) error {
 // Exec the pipeline step.
 func (e *local) Exec(ctx context.Context, step *types.Step) error {
 	// Get environment variables
-	Env := os.Environ()
+	env := os.Environ()
 	for a, b := range step.Environment {
 		if a != "HOME" && a != "SHELL" { // Don't override $HOME and $SHELL
-			Env = append(Env, a+"="+b)
+			env = append(env, a+"="+b)
 		}
 	}
 
-	Command := []string{}
+	command := []string{}
 	if step.Image == constant.DefaultCloneImage {
 		// Default clone step
-		Env = append(Env, "CI_WORKSPACE="+e.workingdir+"/"+step.Environment["CI_REPO"])
-		Command = append(Command, "plugin-git")
+		env = append(env, "CI_WORKSPACE="+e.workingdir+"/"+step.Environment["CI_REPO"])
+		command = append(command, "plugin-git")
 	} else {
 		// Use "image name" as run command
-		Command = append(Command, step.Image)
-		Command = append(Command, "-c")
+		command = append(command, step.Image)
+		command = append(command, "-c")
 
-		// Decode script and delete initial lines
+		// TODO: use commands directly
+		script := common.GenerateScript(step.Commands)
 		// Deleting the initial lines removes netrc support but adds compatibility for more shells like fish
-		Script, _ := base64.RawStdEncoding.DecodeString(step.Environment["CI_SCRIPT"])
-		Command = append(Command, string(Script)[strings.Index(string(Script), "\n\n")+2:])
+		command = append(command, string(script)[strings.Index(string(script), "\n\n")+2:])
 	}
 
 	// Prepare command
-	e.cmd = exec.CommandContext(ctx, Command[0], Command[1:]...)
-	e.cmd.Env = Env
+	e.cmd = exec.CommandContext(ctx, command[0], command[1:]...)
+	e.cmd.Env = env
 
 	// Prepare working directory
 	if step.Image == constant.DefaultCloneImage {

pipeline/backend/ssh/ssh.go

@@ -2,14 +2,14 @@ package ssh
 
 import (
 	"context"
-	"encoding/base64"
 	"fmt"
 	"io"
 	"os"
 	"strings"
 
 	"github.com/melbahja/goph"
 
+	"github.com/woodpecker-ci/woodpecker/pipeline/backend/common"
 	"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
 	"github.com/woodpecker-ci/woodpecker/shared/constant"
 )
@@ -91,31 +91,31 @@ func (e *ssh) Setup(ctx context.Context, config *types.Config) error {
 // Exec the pipeline step.
 func (e *ssh) Exec(ctx context.Context, step *types.Step) error {
 	// Get environment variables
-	Command := []string{}
+	command := []string{}
 	for a, b := range step.Environment {
 		if a != "HOME" && a != "SHELL" { // Don't override $HOME and $SHELL
-			Command = append(Command, a+"="+b)
+			command = append(command, a+"="+b)
 		}
 	}
 
 	if step.Image == constant.DefaultCloneImage {
 		// Default clone step
-		Command = append(Command, "CI_WORKSPACE="+e.workingdir+"/"+step.Environment["CI_REPO"])
-		Command = append(Command, "plugin-git")
+		command = append(command, "CI_WORKSPACE="+e.workingdir+"/"+step.Environment["CI_REPO"])
+		command = append(command, "plugin-git")
 	} else {
 		// Use "image name" as run command
-		Command = append(Command, step.Image)
-		Command = append(Command, "-c")
+		command = append(command, step.Image)
+		command = append(command, "-c")
 
-		// Decode script and delete initial lines
+		// TODO: use commands directly
+		script := common.GenerateScript(step.Commands)
 		// Deleting the initial lines removes netrc support but adds compatibility for more shells like fish
-		Script, _ := base64.RawStdEncoding.DecodeString(step.Environment["CI_SCRIPT"])
-		Command = append(Command, "cd "+e.workingdir+"/"+step.Environment["CI_REPO"]+" && "+string(Script)[strings.Index(string(Script), "\n\n")+2:])
+		command = append(command, "cd "+e.workingdir+"/"+step.Environment["CI_REPO"]+" && "+string(script)[strings.Index(string(script), "\n\n")+2:])
 	}
 
 	// Prepare command
 	var err error
-	e.cmd, err = e.client.CommandContext(ctx, "/bin/env", Command...)
+	e.cmd, err = e.client.CommandContext(ctx, "/bin/env", command...)
 	if err != nil {
 		return err
 	}

pipeline/backend/types/step.go

@@ -12,7 +12,7 @@ type Step struct {
 	Environment  map[string]string `json:"environment,omitempty"`
 	Labels       map[string]string `json:"labels,omitempty"`
 	Entrypoint   []string          `json:"entrypoint,omitempty"`
-	Command      []string          `json:"command,omitempty"`
+	Commands     []string          `json:"commands,omitempty"`
 	ExtraHosts   []string          `json:"extra_hosts,omitempty"`
 	Volumes      []string          `json:"volumes,omitempty"`
 	Tmpfs        []string          `json:"tmpfs,omitempty"`