Add depends_on support for steps

cli/lint/lint.go

@@ -102,9 +102,9 @@ func lintFile(_ *cli.Context, file string) error {
 	// TODO: lint multiple files at once to allow checks for sth like "depends_on" to work
 	err = linter.New(linter.WithTrusted(true)).Lint([]*linter.WorkflowConfig{config})
 	if err != nil {
-		fmt.Printf("🔥 %s has errors:\n", output.String(config.File).Underline())
+		fmt.Printf("🔥 %s has warning / errors:\n", output.String(config.File).Underline())
 
-		hasErrors := true
+		hasErrors := false
 		for _, err := range pipeline_errors.GetPipelineErrors(err) {
 			line := "  "
 

docs/docs/20-usage/20-workflow-syntax.md

@@ -444,33 +444,28 @@ when:
   - evaluate: 'SKIP != "true"'
 ```
 
-### `group` - Parallel execution
+### `depends_on`
 
-Woodpecker supports parallel step execution for same-machine fan-in and fan-out. Parallel steps are configured using the `group` attribute. This instructs the agent to execute the named group in parallel.
-
-Example parallel configuration:
+Normally steps of a workflow are executed serially in the order in which they are defined. As soon as you set `depends_on` for a step a [directed acyclic graph](https://en.wikipedia.org/wiki/Directed_acyclic_graph), all steps of the workflow will be executed in parallel besides the steps that have a dependency set to another step using `depends_on`:
 
 ```diff
  steps:
-   backend:
-+    group: build
-     image: golang
-     commands:
-       - go build
-       - go test
-   frontend:
-+    group: build
-     image: node
-     commands:
-       - npm install
-       - npm run test
-       - npm run build
-   publish:
-     image: plugins/docker
-     repo: octocat/hello-world
-```
+  build: # build will be executed immediately
+    image: golang
+    commands:
+      - go build
+
+  deploy:
+    image: plugins/docker
+    settings:
+      repo: foo/bar
++    depends_on: [build, test] # deploy will be executed after build and test finished
 
-In the above example, the `frontend` and `backend` steps are executed in parallel. The agent will not execute the `publish` step until the group completes.
+  test: # test will be executed immediately as no dependencies are set
+    image: golang
+    commands:
+      - go test
+```
 
 ### `volumes`
 

docs/docs/91-migrations.md

@@ -4,7 +4,7 @@ Some versions need some changes to the server configuration or the pipeline conf
 
 ## `next`
 
-No breaking changes yet
+- Deprecated `steps.[name].group` in favor of `steps.[name].depends_on` (see [workflow syntax](./20-usage/20-workflow-syntax.md#depends_on) to learn how to set dependencies)
 
 ## 2.0.0
 

pipeline/backend/docker/docker.go

@@ -318,10 +318,10 @@
 		for _, step := range stage.Steps {
 			containerName := toContainerName(step)
 			if err := e.client.ContainerKill(noContext, containerName, "9"); err != nil && !isErrContainerNotFoundOrNotRunning(err) {
-				log.Error().Err(err).Msgf("could not kill container '%s'", stage.Name)
+				log.Error().Err(err).Msgf("could not kill container '%s'", step.Name)
 			}
 			if err := e.client.ContainerRemove(noContext, containerName, removeOpts); err != nil && !isErrContainerNotFoundOrNotRunning(err) {
-				log.Error().Err(err).Msgf("could not remove container '%s'", stage.Name)
+				log.Error().Err(err).Msgf("could not remove container '%s'", step.Name)
 			}
 		}
 	}

pipeline/frontend/yaml/compiler/compiler.go

@@ -90,6 +90,11 @@
 	netrcOnlyTrusted  bool
 }
 
+type stepWithDependsOn struct {
+	step      *backend_types.Step
+	dependsOn []string
+}
+
 // New creates a new Compiler with options.
 func New(opts ...Option) *Compiler {
 	compiler := &Compiler{
@@ -234,8 +239,11 @@
 	}
 
 	// add pipeline steps. 1 pipeline step per stage, at the moment
+	stepStages := make([]*backend_types.Stage, 0)
 	var stage *backend_types.Stage
 	var group string
+	steps := make(map[string]*stepWithDependsOn)
+	useDag := false
 	for i, container := range conf.Steps.ContainerList {
 		// Skip if local and should not run local
 		if c.local && !container.When.IsLocal() {
@@ -254,7 +262,11 @@
 			stage = new(backend_types.Stage)
 			stage.Name = fmt.Sprintf("%s_stage_%v", c.prefix, i)
 			stage.Alias = container.Name
-			config.Stages = append(config.Stages, stage)
+			stepStages = append(stepStages, stage)
+		}
+
+		if len(container.DependsOn) > 0 {
+			useDag = true
 		}
 
 		name := fmt.Sprintf("%s_step_%d", c.prefix, i)
@@ -274,9 +286,24 @@
 			}
 		}
 
+		steps[container.Name] = &stepWithDependsOn{
+			step:      step,
+			dependsOn: container.DependsOn,
+		}
 		stage.Steps = append(stage.Steps, step)
 	}
 
+	// dag is used if one or more steps have a depends_on
+	if useDag {
+		stages, err := convertDAGToStages(steps)
+		if err != nil {
+			return nil, err
+		}
+		config.Stages = append(config.Stages, stages...)
+	} else {
+		config.Stages = append(config.Stages, stepStages...)
+	}
+
 	err = c.setupCacheRebuild(conf, config)
 	if err != nil {
 		return nil, err
@@ -328,3 +355,75 @@
 
 	return nil
 }
+
+func dfsVisit(steps map[string]*stepWithDependsOn, name string, visited map[string]struct{}, path []string) error {
+	if _, ok := visited[name]; ok {
+		return fmt.Errorf("cycle detected: %v", path)
+	}
+
+	visited[name] = struct{}{}
+	path = append(path, name)
+
+	for _, dep := range steps[name].dependsOn {
+		if err := dfsVisit(steps, dep, visited, path); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func convertDAGToStages(steps map[string]*stepWithDependsOn) ([]*backend_types.Stage, error) {
+	// Initialize the levels map
+	addedSteps := make(map[string]struct{})
+	stages := make([]*backend_types.Stage, 0)
+
+	for name, step := range steps {
+		// check if all depends_on are valid
+		for _, dep := range step.dependsOn {
+			if _, ok := steps[dep]; !ok {
+				return nil, fmt.Errorf("step %s depends on unknown step %s", name, dep)
+			}
+		}
+
+		// check if there are cycles
+		visited := make(map[string]struct{})
+		if err := dfsVisit(steps, name, visited, []string{}); err != nil {
+			return nil, err
+		}
+	}
+
+	for len(steps) > 0 {
+		addedNodesThisLevel := make(map[string]struct{})
+		stage := &backend_types.Stage{
+			Name:  fmt.Sprintf("stage_%d", len(stages)),
+			Alias: fmt.Sprintf("stage_%d", len(stages)),
+		}
+
+		for name, step := range steps {
+			if allDependenciesSatisfied(step, addedSteps) {
+				stage.Steps = append(stage.Steps, step.step)
+				addedNodesThisLevel[name] = struct{}{}
+				delete(steps, name)
+			}
+		}
+
+		for name := range addedNodesThisLevel {
+			addedSteps[name] = struct{}{}
+		}
+
+		stages = append(stages, stage)
+	}
+
+	return stages, nil
+}
+
+func allDependenciesSatisfied(step *stepWithDependsOn, addedSteps map[string]struct{}) bool {
+	for _, childName := range step.dependsOn {
+		_, ok := addedSteps[childName]
+		if !ok {
+			return false
+		}
+	}
+	return true
+}