Merge branch 'master' into arm-suppport

* master:
  chore: update mergify's titles (elastic#1142)
  chore: remove unused pipelines (elastic#1143)
  feat: use Docker copy to transfer binaries to containers (elastic#1136)
  chore: abstract image pulling (elastic#1137)
  Support multiple deployment backends (elastic#1130)
  chore: remove unused code (elastic#1119)
  Unify fleet and stand-alone suites (elastic#1112)
  Pull fresh docker images before suite (elastic#1123)
  chore: initialise timeout factor next to the declaration (elastic#1118)
  chore: match Go version with Beats (elastic#1120)
  feat: support running k8s autodiscover suite for Beats PRs and local repositories (elastic#1115)
  Auto bootstrap fleet during initialize scenario (elastic#1116)
  Kubernetes Deployment (elastic#1110)

.go-version

@@ -1 +1 @@
-1.14.12
+1.15.9

.mergify.yml

@@ -24,6 +24,7 @@ pull_request_rules:
           - "{{ author }}"
         branches:
           - "7.x"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
   - name: backport patches to 7.13 branch
     conditions:
       - merged
@@ -35,6 +36,7 @@ pull_request_rules:
           - "{{ author }}"
         branches:
           - "7.13.x"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
   - name: backport patches to 7.12 branch
     conditions:
       - merged
@@ -46,3 +48,4 @@ pull_request_rules:
           - "{{ author }}"
         branches:
           - "7.12.x"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"

.pre-commit-config.yaml

@@ -8,6 +8,8 @@ repos:
         exclude: ^notice/overrides.json
     -   id: check-merge-conflict
     -   id: check-yaml
+        exclude: >
+          (?x)^(cli/config/kubernetes.*)$
     -   id: check-xml
     -   id: end-of-file-fixer
         exclude: >
@@ -33,15 +35,23 @@ repos:
     -   id: check-unicode-non-breaking-spaces
         exclude: >
             (?x)^(
-                .*/?\.*.(gif|jpg|png)
+                .*/?\.*.(gif|jpg|png|tar.gz)
             )$
     -   id: remove-unicode-non-breaking-spaces
         exclude: >
             (?x)^(
-                .*/?\.*.(gif|jpg|png)
+                .*/?\.*.(gif|jpg|png|tar.gz)
             )$
     -   id: check-en-dashes
+        exclude: >
+            (?x)^(
+                .*/?\.*.(tar.gz)
+            )$
     -   id: remove-en-dashes
+        exclude: >
+            (?x)^(
+                .*/?\.*.(tar.gz)
+            )$
     -   id: check-jjbb
     -   id: check-gherkin-lint
         args: [

cli/config/compose/profiles/fleet/configurations/kibana.config.yml

@@ -16,5 +16,5 @@ xpack.fleet.registryUrl: https://epr-staging.elastic.co
 xpack.fleet.agents.enabled: true
 xpack.fleet.agents.elasticsearch.host: http://elasticsearch:9200
 xpack.fleet.agents.fleet_server.hosts:
-  - http://kibana:5601
+  - http://fleet-server:8220
 xpack.fleet.agents.tlsCheckDisabled: true

cli/config/compose/profiles/fleet/docker-compose.yml

@@ -34,3 +34,21 @@ services:
       - "5601:5601"
     volumes:
       - ./configurations/kibana.config.yml:/usr/share/kibana/config/kibana.yml
+  fleet-server:
+    image: "docker.elastic.co/beats/elastic-agent:${stackVersion:-8.0.0-SNAPSHOT}"
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+      kibana:
+        condition: service_healthy
+    healthcheck:
+      test: "curl -f http://127.0.0.1:8220/api/status | grep HEALTHY 2>&1 >/dev/null"
+      retries: 12
+      interval: 5s
+    environment:
+      - "FLEET_SERVER_ENABLE=1"
+      - "FLEET_SERVER_INSECURE_HTTP=1"
+      - "KIBANA_FLEET_SETUP=1"
+      - "KIBANA_FLEET_HOST=http://kibana:5601"
+      - "FLEET_SERVER_HOST=0.0.0.0"
+      - "FLEET_SERVER_PORT=8220"

cli/config/compose/services/centos-systemd/docker-compose.yml

@@ -7,5 +7,4 @@ services:
     platform: ${stackPlatform:-linux/amd64}
     privileged: true
     volumes:
-      - ${centos_systemdAgentBinarySrcPath:-.}:${centos_systemdAgentBinaryTargetPath:-/tmp}
       - /sys/fs/cgroup:/sys/fs/cgroup:ro 

cli/config/compose/services/centos/docker-compose.yml

@@ -4,5 +4,3 @@ services:
     image: centos:${centosTag:-7}
     container_name: ${centosContainerName} 
     entrypoint: tail -f /dev/null
-    volumes:
-      - ${centosAgentBinarySrcPath:-.}:${centosAgentBinaryTargetPath:-/tmp}

cli/config/compose/services/debian-systemd/docker-compose.yml

@@ -7,5 +7,4 @@ services:
     platform: ${stackPlatform:-linux/amd64}
     privileged: true
     volumes:
-      - ${debian_systemdAgentBinarySrcPath:-.}:${debian_systemdAgentBinaryTargetPath:-/tmp}
       - /sys/fs/cgroup:/sys/fs/cgroup:ro

cli/config/compose/services/debian/docker-compose.yml

@@ -4,5 +4,3 @@ services:
     image: debian:${debianTag:-9}
     container_name: ${debianContainerName} 
     entrypoint: tail -f /dev/null
-    volumes:
-      - ${debianAgentBinarySrcPath:-.}:${debianAgentBinaryTargetPath:-/tmp}

cli/config/compose/services/fleet-server-centos/docker-compose.yml

@@ -7,5 +7,4 @@ services:
     platform: ${stackPlatform:-linux/amd64}
     privileged: true
     volumes:
-      - ${fleet_server_centosAgentBinarySrcPath:-.}:${fleet_server_centosAgentBinaryTargetPath:-/tmp}
       - /sys/fs/cgroup:/sys/fs/cgroup:ro 

cli/config/compose/services/fleet-server-debian/docker-compose.yml

@@ -7,5 +7,4 @@ services:
     platform: ${stackPlatform:-linux/amd64}
     privileged: true
     volumes:
-      - ${fleet_server_debianAgentBinarySrcPath:-.}:${fleet_server_debianAgentBinaryTargetPath:-/tmp}
       - /sys/fs/cgroup:/sys/fs/cgroup:ro

cli/config/kubernetes/README.md

@@ -0,0 +1,26 @@
+# K8s deployment for Elasticsearch, Kibana, and Fleet-Server
+
+## Requirements
+
+- docker
+- kind (>= 0.10.0)
+- kubectl (>= 1.17)
+
+## Deployment
+
+```
+kind create cluster
+kubectl apply -k base
+```
+
+This will allow you to visit `http://localhost:5601` using username: `elastic` password: `changeme` to login
+
+## Local Ingress (Optional)
+
+If a local ingress is preferred, apply the following to setup nginx-ingress
+
+```
+kubectl apply -k overlays/local
+```
+
+This will allow you to reach the Kibana endpoint at `http://localhost`

cli/config/kubernetes/base/elasticsearch/configmap.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: elasticsearch-config
+data:
+  ES_JAVA_OPTS: "-Xms1g -Xmx1g"
+  network.host: ""
+  transport.host: "127.0.0.1"
+  http.host: "0.0.0.0"
+  indices.id_field_data.enabled: 'true'
+  xpack.license.self_generated.type: "trial"
+  xpack.security.enabled: 'true'
+  xpack.security.authc.api_key.enabled: 'true'
+  ELASTIC_USERNAME: "elastic"
+  ELASTIC_PASSWORD: "changeme"

cli/config/kubernetes/base/elasticsearch/deployment.yaml

@@ -0,0 +1,25 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: elasticsearch
+  labels:
+    app: elasticsearch
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: elasticsearch
+  template:
+    metadata:
+      labels:
+        app: elasticsearch
+    spec:
+      containers:
+      - name: elasticsearch
+        image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0-SNAPSHOT
+        envFrom:
+          - configMapRef:
+              name: elasticsearch-config
+        ports:
+        - containerPort: 9200
+          name: client

cli/config/kubernetes/base/elasticsearch/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+- deployment.yaml
+- service.yaml
+- configmap.yaml

cli/config/kubernetes/base/elasticsearch/service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch
+  labels:
+    service: elasticsearch
+spec:
+  type: NodePort
+  ports:
+  - port: 9200
+    name: client
+  selector:
+    app: elasticsearch

cli/config/kubernetes/base/fleet-server/cluster-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: fleet-server-cluster-role-binding
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: default
+roleRef:
+  kind: ClusterRole
+  name: fleet-server-cluster-role
+  apiGroup: rbac.authorization.k8s.io