feat: metricbeat step (#1050)

* feat: metricbeat step

* fix:tests

* feat: use resources for scripts and configuration files

* fix: rever change on testMissingArgument

* docs: update REAMDE

resources/scripts/beats/metricbeat.yml

@@ -0,0 +1,36 @@
+---
+metricbeat.modules:
+  - module: system
+    metricsets:
+      - cpu             # CPU usage
+      - load            # CPU load averages
+      - memory          # Memory usage
+      - network         # Network IO
+      - process         # Per process metrics
+      - process_summary # Process summary
+      - uptime          # System Uptime
+      - socket_summary  # Socket summary
+      #- core           # Per CPU core usage
+      #- diskio         # Disk IO
+      #- filesystem     # File system usage for each mountpoint
+      #- fsstat         # File system summary metrics
+      #- raid           # Raid
+      #- socket         # Sockets and connection info (linux only)
+      #- service        # systemd service information
+    enabled: true
+    period: 10s
+    processes: ['.*']
+
+    # Configure the metric types that are included by these metricsets.
+    cpu.metrics: ["percentages", "normalized_percentages"] # The other available option is ticks.
+    core.metrics: ["percentages"] # The other available option is ticks.
+
+processors:
+  - add_host_metadata: ~
+  - add_cloud_metadata: ~
+  - add_docker_metadata: ~
+  - add_kubernetes_metadata: ~
+output.elasticsearch:
+  hosts: ["${ES_URL}"]
+  username: "${ES_USERNAME}"
+  password: "${ES_PASSWORD}"

resources/scripts/beats/run_metricbeat.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -eu
+docker run \
+  --detach \
+  -v "${CONFIG_PATH}:/usr/share/metricbeat/metricbeat.yml" \
+  -u 0:0 \
+  --mount type=bind,source=/proc,target=/hostfs/proc,readonly \
+  --mount type=bind,source=/sys/fs/cgroup,target=/hostfs/sys/fs/cgroup,readonly \
+  --mount type=bind,source=/,target=/hostfs,readonly \
+  --net=host \
+  -e ES_URL="${ES_URL}" \
+  -e ES_USERNAME="${ES_USERNAME}" \
+  -e ES_PASSWORD="${ES_PASSWORD}" \
+  "${DOCKER_IMAGE}" \
+    --strict.perms=false \
+    -environment container \
+    -E http.enabled=true \
+    -e -system.hostfs=/hostfs > docker_id

src/test/groovy/ApmBasePipelineTest.groovy

@@ -667,6 +667,17 @@ class ApmBasePipelineTest extends DeclarativePipelineTest {
     assertJobStatusFailure()
   }
 
+  def testError(String message='parameter is required', Closure body) {
+    try {
+      body()
+    } catch(e){
+      //NOOP
+    }
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('error', "${message}"))
+    assertJobStatusFailure()
+  }
+
   def testWindows(Closure body) {
     helper.registerAllowedMethod('isUnix', [], { false })
     try {

src/test/groovy/MetricbeatStepTests.groovy

@@ -0,0 +1,191 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Before
+import org.junit.Test
+import static org.junit.Assert.assertTrue
+import static org.junit.Assert.assertFalse
+
+class MetricbeatStepTests extends ApmBasePipelineTest {
+  // test resources file uses this value as filename
+  String nodeName = 'worker-0676d01d9601f8191'
+  String jsonConfig = "metricbeat_container_" + nodeName + ".json"
+  String resources = "target/test-classes"
+
+  @Override
+  @Before
+  void setUp() throws Exception {
+    super.setUp()
+    script = loadScript('vars/metricbeat.groovy')
+    env.NODE_NAME = nodeName
+    env.WORKSPACE = "metricbeatTest"
+    helper.registerAllowedMethod('writeFile', [Map.class], { m ->
+      (new File("${resources}/${m.file}")).withWriter('UTF-8') { writer ->
+        writer.write(m.text)
+      }
+    })
+    helper.registerAllowedMethod('writeJSON', [Map.class], { m ->
+      def script = loadScript('vars/toJSON.groovy')
+      def json =  script.call(m.json)
+      (new File("${resources}/${m.file}")).withWriter('UTF-8') { writer ->
+        writer.write(json.toString())
+      }
+    })
+    helper.registerAllowedMethod('readJSON', [Map.class], { m ->
+      def jsonSlurper = new groovy.json.JsonSlurperClassic()
+      File f = new File("${resources}/${m.file}")
+      jsonText = f.getText()
+      return jsonSlurper.parseText(jsonText)
+    })
+    helper.registerAllowedMethod('sh', [Map.class], { m ->
+      def ret = "OK"
+      if(m.script.contains('docker run')){
+        ret = 'fooID'
+      }
+      return ret
+    })
+    helper.registerAllowedMethod('pwd', [], { 'metricbeatTest' })
+    helper.registerAllowedMethod('isBuildFailure', [], { false })
+    helper.registerAllowedMethod('readFile', [Map.class], { 'fooID' })
+  }
+
+  @Test
+  void test() throws Exception {
+    helper.registerAllowedMethod('fileExists', [String.class], { false })
+    printCallStack(){
+      script.call(es_secret: 'foo')
+    }
+    assertTrue(assertMethodCallContainsPattern('writeFile', "file=metricbeatTest/metricbeat_conf.yml"))
+    assertTrue(assertMethodCallContainsPattern('sh', 'run_metricbeat.sh'))
+    assertJobStatusSuccess()
+  }
+
+  @Test
+  void testNoSecret() throws Exception {
+    helper.registerAllowedMethod('fileExists', [String.class], { false })
+    testError('metricbeat: The parameter es_secret is mandatory.'){
+      script.call()
+    }
+  }
+
+  @Test
+  void testClosure() throws Exception {
+    def id = "fooID"
+    def workdir = "metricbeatTest_1"
+    def config = "bar.xml"
+    def image = "foo:latest"
+
+    helper.registerAllowedMethod('fileExists', [String.class], { f ->
+      if(f == "${workdir}/${config}"){
+        return false
+      } else {
+        return true
+      }
+    })
+
+    printCallStack(){
+      script.call(
+        es_secret: 'foo',
+        config: config,
+        image: image,
+        workdir: workdir,
+        timeout: "30",
+        ){
+        print("OK")
+      }
+    }
+
+    assertTrue(assertMethodCallContainsPattern('writeFile', "file=${workdir}/${config}"))
+    assertTrue(assertMethodCallContainsPattern('readJSON', "file=${workdir}/${jsonConfig}"))
+    assertTrue(assertMethodCallContainsPattern('sh', "docker stop --time 30 ${id}"))
+    assertJobStatusSuccess()
+  }
+
+  @Test(expected = Exception.class)
+  void testClosureError() throws Exception {
+    def id = "fooID"
+    def output = "foo.log"
+    def workdir = "metricbeatTest_1"
+    def config = "bar.xml"
+    def image = "foo:latest"
+
+    helper.registerAllowedMethod('fileExists', [String.class], { f ->
+      if(f == "${workdir}/${config}"){
+        return false
+      } else {
+        return true
+      }
+    })
+
+    try {
+      script.call(
+        es_secret: 'foo',
+        config: config,
+        image: image,
+        workdir: workdir,
+        timeout: "30",
+        ){
+        throw new Exception('Ooops!!')
+      }
+    } finally {
+      printCallStack()
+      assertTrue(assertMethodCallContainsPattern('writeFile', "file=${workdir}/${config}"))
+      assertTrue(assertMethodCallContainsPattern('readJSON', "file=${workdir}/${jsonConfig}"))
+      assertTrue(assertMethodCallContainsPattern('sh', 'run_metricbeat.sh'))
+      assertTrue(assertMethodCallContainsPattern('sh', "docker stop --time 30 ${id}"))
+    }
+  }
+
+  @Test
+  void testConfigurationExists() throws Exception {
+    printCallStack(){
+      script.call(es_secret: 'foo')
+    }
+    assertTrue(assertMethodCallContainsPattern('sh', 'run_metricbeat.sh'))
+    assertFalse(assertMethodCallContainsPattern('writeFile', 'file=metricbeat_conf.yml'))
+    assertJobStatusSuccess()
+  }
+
+  @Test
+  void testStop() throws Exception {
+    def id = "fooID"
+    def output = "foo.log"
+    def workdir = "metricbeatTest"
+
+    printCallStack(){
+      script.stop(
+        workdir: workdir,
+      )
+    }
+    assertTrue(assertMethodCallContainsPattern('readJSON', "file=${workdir}/${jsonConfig}"))
+    assertTrue(assertMethodCallContainsPattern('sh', "docker stop --time 30 ${id}"))
+    assertJobStatusSuccess()
+  }
+
+  @Test
+  void testConfigFileNotExists() throws Exception {
+    helper.registerAllowedMethod('fileExists', [String.class], { false })
+
+    def workdir = "metricbeatTest_1"
+
+    printCallStack(){
+      script.stop(workdir: workdir)
+    }
+    assertTrue(assertMethodCallContainsPattern('log', "There is no configuration file to stop metricbeat."))
+    assertJobStatusSuccess()
+  }
+}

src/test/resources/metricbeatTest/metricbeat_container_worker-0676d01d9601f8191.json

@@ -0,0 +1,7 @@
+{
+  "id": "fooID",
+  "config": "bar.xml",
+  "image": "foo: latest",
+  "workdir": "fooDir",
+  "timeout": "30"
+}

src/test/resources/metricbeatTest_2/metricbeat_container_worker-0676d01d9601f8191.json

@@ -0,0 +1,8 @@
+{
+  "id": "fooID",
+  "config": "bar.xml",
+  "image": "foo: latest",
+  "workdir": "fooDir",
+  "timeout": "30",
+  "archiveOnlyOnFail": "true"
+}

vars/README.md

@@ -1739,6 +1739,77 @@ pipeline {
 
 ```
 
+## metricbeat
+
+ This step run a filebeat Docker container to grab the Docker containers logs in a single file.
+ `filebeat.stop()` will stop the Filebeat Docker container and grab the output files,
+ the only argument need is the `workdir` if you set it on the `filebeat step` call.
+ The output log files should be in a relative path to the current path (see [archiveArtifacts](https://www.jenkins.io/doc/pipeline/steps/core/#archiveartifacts-archive-the-artifacts))
+
+```
+  filebeat()
+  ...
+  filebeat.stop()
+```
+
+```
+  filebeat(){
+    ....
+  }
+```
+
+* *config:* Filebeat configuration file, a default configuration is created if the file does not exists (filebeat_conf.yml).
+* *image:* Filebeat Docker image to use (docker.elastic.co/beats/filebeat:7.10.1).
+* *output:* log file to save all Docker containers logs (docker_logs.log).
+* *timeout:* Time to wait before kill the Filebeat Docker container on the stop operation.
+* *workdir:* Directory to use as root folder to read and write files (current folder).
+* *archiveOnlyOnFail:* if true only archive the files in case of failure.
+
+```
+  filebeat(config: 'filebeat.yml',
+    image: 'docker.elastic.co/beats/filebeat:7.10.1',
+    output: 'docker_logs.log',
+    workdir: "${env.WORKSPACE}")
+  ...
+  filebeat.stop(workdir: "${env.WORKSPACE}")
+```
+
+```
+pipeline {
+  agent { label "ubuntu" }
+  stages {
+    stage('My Docker tests') {
+      steps {
+        filebeat(workdir: "${env.WORKSPACE}")
+        sh('docker run busybox  ls')
+      }
+      post {
+        cleanup{
+          script {
+            filebeat.stop(workdir: "${env.WORKSPACE}")
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+```
+pipeline {
+  agent { label "ubuntu" }
+  stages {
+    stage('My Docker tests') {
+      steps {
+        filebeat(workdir: "${env.WORKSPACE}"){
+          sh('docker run busybox  ls')
+        }
+      }
+    }
+  }
+}
+```
+
 ## mvnVersion
 Get a project version from Maven