Add agent tagging / filtering for pipelines (#902)

Officially support labels for pipelines and agents to improve pipeline picking. 

* add pipeline labels
* update, improve docs  and add migration
* update proto file

---
closes #304 & #860

agent/runner.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"io"
 	"io/ioutil"
+	"runtime"
 	"strconv"
 	"sync"
 	"time"
@@ -283,7 +284,7 @@ func (r *Runner) Run(ctx context.Context) error {
 			state.Pipeline.Step.Environment = map[string]string{}
 		}
 
-		// TODO: find better way to update this state
+		// TODO: find better way to update this state and move it to pipeline to have the same env in cli-exec
 		state.Pipeline.Step.Environment["CI_MACHINE"] = r.hostname
 		state.Pipeline.Step.Environment["CI_BUILD_STATUS"] = "success"
 		state.Pipeline.Step.Environment["CI_BUILD_STARTED"] = strconv.FormatInt(state.Pipeline.Time, 10)
@@ -293,6 +294,8 @@ func (r *Runner) Run(ctx context.Context) error {
 		state.Pipeline.Step.Environment["CI_JOB_STARTED"] = strconv.FormatInt(state.Pipeline.Time, 10)
 		state.Pipeline.Step.Environment["CI_JOB_FINISHED"] = strconv.FormatInt(time.Now().Unix(), 10)
 
+		state.Pipeline.Step.Environment["CI_SYSTEM_ARCH"] = runtime.GOOS + "/" + runtime.GOARCH
+
 		if state.Pipeline.Error != nil {
 			state.Pipeline.Step.Environment["CI_BUILD_STATUS"] = "failure"
 			state.Pipeline.Step.Environment["CI_JOB_STATUS"] = "failure"

cli/exec/exec.go

@@ -206,6 +206,11 @@ func execWithAxis(c *cli.Context, file, repoPath string, axis matrix.Axis) error
 
 // return the metadata from the cli context.
 func metadataFromContext(c *cli.Context, axis matrix.Axis) frontend.Metadata {
+	platform := c.String("system-platform")
+	if platform == "" {
+		platform = runtime.GOOS + "/" + runtime.GOARCH
+	}
+
 	return frontend.Metadata{
 		Repo: frontend.Repo{
 			Name:    c.String("repo-name"),
@@ -262,9 +267,9 @@ func metadataFromContext(c *cli.Context, axis matrix.Axis) frontend.Metadata {
 			Matrix: axis,
 		},
 		Sys: frontend.System{
-			Name: c.String("system-name"),
-			Link: c.String("system-link"),
-			Arch: c.String("system-arch"),
+			Name:     c.String("system-name"),
+			Link:     c.String("system-link"),
+			Platform: platform,
 		},
 	}
 }

cli/exec/flags.go

@@ -101,9 +101,8 @@ var flags = []cli.Flag{
 	// metadata parameters
 	//
 	&cli.StringFlag{
-		EnvVars: []string{"CI_SYSTEM_ARCH"},
-		Name:    "system-arch",
-		Value:   "linux/amd64",
+		EnvVars: []string{"CI_SYSTEM_PLATFORM"},
+		Name:    "system-platform",
 	},
 	&cli.StringFlag{
 		EnvVars: []string{"CI_SYSTEM_NAME"},

cmd/agent/agent.go

@@ -20,6 +20,7 @@ import (
 	"net/http"
 	"os"
 	"runtime"
+	"strings"
 	"sync"
 
 	"github.com/rs/zerolog"
@@ -39,18 +40,26 @@ import (
 )
 
 func loop(c *cli.Context) error {
-	filter := rpc.Filter{
-		Labels: map[string]string{
-			"platform": runtime.GOOS + "/" + runtime.GOARCH,
-		},
-		Expr: c.String("filter"),
-	}
-
 	hostname := c.String("hostname")
 	if len(hostname) == 0 {
 		hostname, _ = os.Hostname()
 	}
 
+	labels := map[string]string{
+		"hostname": hostname,
+		"platform": runtime.GOOS + "/" + runtime.GOARCH,
+		"repo":     "*", // allow all repos by default
+	}
+
+	for _, v := range c.StringSlice("filter-labels") {
+		parts := strings.SplitN(v, "=", 2)
+		labels[parts[0]] = parts[1]
+	}
+
+	filter := rpc.Filter{
+		Labels: labels,
+	}
+
 	if c.Bool("pretty") {
 		log.Logger = log.Output(
 			zerolog.ConsoleWriter{

cmd/agent/flags.go

@@ -72,10 +72,10 @@ var flags = []cli.Flag{
 		Name:    "hostname",
 		Usage:   "agent hostname",
 	},
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_FILTER"},
+	&cli.StringSliceFlag{
+		EnvVars: []string{"WOODPECKER_FILTER_LABELS"},
 		Name:    "filter",
-		Usage:   "filter expression to restrict builds by label",
+		Usage:   "List of labels to filter tasks on. An agent must be assigned every tag listed in a task to be selected.",
 	},
 	&cli.IntFlag{
 		EnvVars: []string{"WOODPECKER_MAX_PROCS"},

docs/docs/20-usage/20-pipeline-syntax.md

@@ -96,26 +96,6 @@ pipeline:
 +  exclude: [ develop, feature/* ]
 ```
 
-### `platform`
-
-To configure your pipeline to only be executed on an agent with a specific platform, you can use the `platform` key.
-Have a look at the official [go docs](https://go.dev/doc/install/source) for the available platforms. The syntax of the platform is `GOOS/GOARCH` like `linux/arm64` or `linux/amd64`.
-
-Example:
-
-Assuming we have two agents, one `arm` and one `amd64`. Previously this pipeline would have executed on **either agent**, as Woodpecker is not fussy about where it runs the pipelines. By setting the following option it will only be executed on an agent with the platform `linux/arm64`.
-
-```diff
-+platform: linux/arm64
-
- pipeline:
-   build:
-     image: golang
-     commands:
-       - go build
-       - go test
-```
-
 ### Skip Commits
 
 Woodpecker gives the ability to skip individual commits by adding `[CI SKIP]` to the commit message. Note this is case-insensitive.
@@ -401,6 +381,10 @@ pipeline:
 
 #### `platform`
 
+:::note
+This condition should be used in conjunction with a [matrix](/docs/usage/matrix-pipelines#example-matrix-pipeline-using-multiple-platforms) pipeline as a regular pipeline will only executed by a single agent which only has one arch.
+:::
+
 Execute a step for a specific platform:
 
 ```diff
@@ -448,8 +432,8 @@ when:
 #### `path`
 
 :::info
-Path conditions are applied only to **push** and **pull_request** events.  
-It is currently **only available** for GitHub, GitLab.  
+Path conditions are applied only to **push** and **pull_request** events.
+It is currently **only available** for GitHub, GitLab.
 Gitea only support **push** at the moment ([go-gitea/gitea#18228](https://github.com/go-gitea/gitea/pull/18228)).
 :::
 
@@ -586,7 +570,52 @@ git clone https://github.com/octocat/hello-world \
 
 Woodpecker has integrated support for matrix builds. Woodpecker executes a separate build task for each combination in the matrix, allowing you to build and test a single commit against multiple configurations.
 
-For more details check the [matrix build docs](/docs/usage/matrix-builds/).
+For more details check the [matrix build docs](/docs/usage/matrix-pipelines/).
+
+## `platform`
+
+To configure your pipeline to only be executed on an agent with a specific platform, you can use the `platform` key.
+Have a look at the official [go docs](https://go.dev/doc/install/source) for the available platforms. The syntax of the platform is `GOOS/GOARCH` like `linux/arm64` or `linux/amd64`.
+
+Example:
+
+Assuming we have two agents, one `arm` and one `amd64`. Previously this pipeline would have executed on **either agent**, as Woodpecker is not fussy about where it runs the pipelines. By setting the following option it will only be executed on an agent with the platform `linux/arm64`.
+
+```diff
++platform: linux/arm64
+
+pipeline:
+  build:
+    image: golang
+    commands:
+      - go build
+      - go test
+```
+
+## `labels`
+
+You can set labels for your pipeline to select an agent to execute the pipeline on. An agent will pick up and run a pipeline when **every** label assigned to a pipeline matches the agents labels.
+
+To set additional agent labels check the [agent configuration options](/docs/administration/agent-config#woodpecker_filter_labels). Agents will have at least three default labels: `platform=agent-os/agent-arch`, `hostname=my-agent` and `repo=*`. Agents can use a `*` as a wildcard for a label. For example `repo=*` will match every repo.
+
+Pipeline labels with an empty value will be ignored.
+By default each pipeline has at least the `repo=your-user/your-repo-name` label. If you have set the [platform attribute](#platform) for your pipeline it will have a label like `platform=your-os/your-arch` as well.
+
+You can add additional labels as a key value map:
+
+```diff
++labels:
++  location: europe # only agents with `location=europe` or `location=*` will be used
++  weather: sun
++  hostname: "" # this label will be ignored as it is empty
+
+pipeline:
+  build:
+    image: golang
+    commands:
+      - go build
+      - go test
+```
 
 ## `clone`
 

docs/docs/20-usage/30-matrix-builds.md → docs/docs/20-usage/30-matrix-pipelines.md

@@ -1,6 +1,6 @@
-# Matrix builds
+# Matrix pipelines
 
-Woodpecker has integrated support for matrix builds. Woodpecker executes a separate build task for each combination in the matrix, allowing you to build and test a single commit against multiple configurations.
+Woodpecker has integrated support for matrix pipeline. Woodpecker executes a separate pipeline for each combination in the matrix, allowing you to build and test a single commit against multiple configurations.
 
 Example matrix definition:
 
@@ -33,6 +33,15 @@ matrix:
 Matrix variables are interpolated in the yaml using the `${VARIABLE}` syntax, before the yaml is parsed. This is an example yaml file before interpolating matrix parameters:
 
 ```yaml
+matrix:
+  GO_VERSION:
+    - 1.4
+    - 1.3
+  DATABASE:
+    - mysql:5.5
+    - mysql:6.5
+    - mariadb:10.1
+
 pipeline:
   build:
     image: golang:${GO_VERSION}
@@ -44,15 +53,6 @@ pipeline:
 services:
   database:
     image: ${DATABASE}
-
-matrix:
-  GO_VERSION:
-    - 1.4
-    - 1.3
-  DATABASE:
-    - mysql:5.5
-    - mysql:6.5
-    - mariadb:10.1
 ```
 
 Example YAML file after injecting the matrix parameters:
@@ -78,36 +78,61 @@ services:
 
 ## Examples
 
-Example matrix build based on Docker image tag:
+### Example matrix pipeline based on Docker image tag
 
 ```yaml
+matrix:
+  TAG:
+    - 1.7
+    - 1.8
+    - latest
+
 pipeline:
   build:
     image: golang:${TAG}
     commands:
       - go build
       - go test
-
-matrix:
-  TAG:
-    - 1.7
-    - 1.8
-    - latest
 ```
 
-Example matrix build based on Docker image:
+### Example matrix pipeline based on container image
 
 ```yaml
+matrix:
+  IMAGE:
+    - golang:1.7
+    - golang:1.8
+    - golang:latest
+
 pipeline:
   build:
     image: ${IMAGE}
     commands:
       - go build
       - go test
+```
 
+### Example matrix pipeline using multiple platforms
+
+```yaml
 matrix:
-  IMAGE:
-    - golang:1.7
-    - golang:1.8
-    - golang:latest
+  platform:
+    - linux/amd64
+    - linux/arm64
+
+platform: ${platform}
+
+pipeline:
+  test:
+    image: alpine
+    commands:
+      - echo "I am running on ${platform}"
+
+  test-arm-only:
+    image: alpine
+    commands:
+      - echo "I am running on ${platform}"
+      - echo "Arm is cool!"
+    when:
+      platform: linux/arm*
 ```