Reusing module config collector script across Filebeat and Metricbeat

filebeat/Makefile

@@ -6,6 +6,8 @@ GOX_FLAGS=-arch="amd64 386 arm ppc64 ppc64le"
 ES_BEATS?=..
 FIELDS_FILE_PATH=module
 
+DOCS_BRANCH=$(shell grep doc-branch ../libbeat/docs/version.asciidoc | cut -c 14-)
+
 include ${ES_BEATS}/libbeat/scripts/Makefile
 
 # Collects all module dashboards
@@ -23,8 +25,8 @@ configs: python-env
 	@cat ${ES_BEATS}/filebeat/_meta/common.reference.p1.yml > _meta/beat.reference.yml
 	@${PYTHON_ENV}/bin/python  ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.reference.yml
 	@cat ${ES_BEATS}/filebeat/_meta/common.reference.p2.yml >> _meta/beat.reference.yml
-	@rm -rf modules.d && mkdir -p modules.d
-	@for MODULE in `find module -type d -maxdepth 1 -mindepth 1 -exec basename {} \;`; do cp -a $(PWD)/module/$$MODULE/_meta/config.yml modules.d/$$MODULE.yml.disabled; done
+	@rm -rf modules.d
+	${PYTHON_ENV}/bin/python ${ES_BEATS}/script/modules_collector.py --beat ${BEAT_NAME} --docs_branch=$(DOCS_BRANCH)
 	@chmod go-w modules.d/*
 
 # Collects all module docs

filebeat/modules.d/apache2.yml.disabled

@@ -1,3 +1,6 @@
+# Module: apache2
+# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-apache2.html
+
 - module: apache2
   # Access logs
   access:

filebeat/modules.d/auditd.yml.disabled

@@ -1,3 +1,6 @@
+# Module: auditd
+# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-auditd.html
+
 - module: auditd
   log:
     enabled: true

filebeat/modules.d/elasticsearch.yml.disabled

@@ -1,3 +1,6 @@
+# Module: elasticsearch
+# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-elasticsearch.html
+
 - module: elasticsearch
   # Server log
   server:

filebeat/modules.d/haproxy.yml.disabled

@@ -1,3 +1,6 @@
+# Module: haproxy
+# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-haproxy.html
+
 - module: haproxy
   # All logs
   log:

filebeat/modules.d/icinga.yml.disabled

@@ -1,3 +1,6 @@
+# Module: icinga
+# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-icinga.html
+
 - module: icinga
   # Main logs
   main:

filebeat/modules.d/iis.yml.disabled

@@ -1,3 +1,6 @@
+# Module: iis
+# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-iis.html
+
 - module: iis
   # Access logs
   access:

filebeat/modules.d/kafka.yml.disabled

@@ -1,3 +1,6 @@
+# Module: kafka
+# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-kafka.html
+
 - module: kafka
   # All logs
   log:

filebeat/modules.d/kibana.yml.disabled

@@ -1,3 +1,6 @@
+# Module: kibana
+# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-kibana.html
+
 - module: kibana
   # All logs
   log:

filebeat/modules.d/logstash.yml.disabled

@@ -1,3 +1,6 @@
+# Module: logstash
+# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-logstash.html
+
 - module: logstash
   # logs
   log:

filebeat/modules.d/mongodb.yml.disabled

@@ -1,3 +1,6 @@
+# Module: mongodb
+# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-mongodb.html
+
 - module: mongodb
   # All logs
   log:

filebeat/modules.d/mysql.yml.disabled

@@ -1,3 +1,6 @@
+# Module: mysql
+# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-mysql.html
+
 - module: mysql
   # Error logs
   error:

filebeat/modules.d/nginx.yml.disabled

@@ -1,3 +1,6 @@
+# Module: nginx
+# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-nginx.html
+
 - module: nginx
   # Access logs
   access:

filebeat/modules.d/osquery.yml.disabled

@@ -1,3 +1,6 @@
+# Module: osquery
+# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-osquery.html
+
 - module: osquery
   result:
     enabled: true

filebeat/modules.d/postgresql.yml.disabled

@@ -1,3 +1,6 @@
+# Module: postgresql
+# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-postgresql.html
+
 - module: postgresql
   # All logs
   log:

filebeat/modules.d/redis.yml.disabled

@@ -1,3 +1,6 @@
+# Module: redis
+# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-redis.html
+
 - module: redis
   # Main logs
   log:

filebeat/modules.d/system.yml.disabled

@@ -1,3 +1,6 @@
+# Module: system
+# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-system.html
+
 - module: system
   # Syslog
   syslog:

filebeat/modules.d/traefik.yml.disabled

@@ -1,3 +1,6 @@
+# Module: traefik
+# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-traefik.html
+
 - module: traefik
   # Access logs
   access:

metricbeat/Makefile

@@ -36,7 +36,7 @@ configs: python-env
 	@cat ${ES_BEATS}/metricbeat/_meta/common.reference.yml > _meta/beat.reference.yml
 	@${PYTHON_ENV}/bin/python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.reference.yml
 	@rm -rf modules.d
-	${PYTHON_ENV}/bin/python ${ES_BEATS}/metricbeat/scripts/modules_collector.py --docs_branch=$(DOCS_BRANCH)
+	${PYTHON_ENV}/bin/python ${ES_BEATS}/script/modules_collector.py --beat ${BEAT_NAME} --docs_branch=$(DOCS_BRANCH)
 	@chmod go-w modules.d/*
 	@# Enable system by default:
 	@if [ -f modules.d/system.yml.disabled ]; then mv modules.d/system.yml.disabled modules.d/system.yml; fi

script/modules_collector.py

@@ -0,0 +1,61 @@
+import os
+import argparse
+import yaml
+import six
+import glob
+import re
+
+# Collects module configs to modules.d
+
+REFERENCE_CONFIG_RE = re.compile('.+\.reference\.yml')
+
+
+def collect(beat_name, docs_branch):
+
+    base_dir = "module"
+    path = os.path.abspath("module")
+
+    # TODO add module release status if beta or experimental
+    header = """# Module: {module}
+# Docs: https://www.elastic.co/guide/en/beats/{beat_name}/{docs_branch}/{beat_name}-module-{module}.html
+
+"""
+
+    # Create directory for module confs
+    os.mkdir(os.path.abspath('modules.d'))
+
+    # Iterate over all modules
+    for module in sorted(os.listdir(base_dir)):
+
+        module_confs = path + '/' + module + '/_meta/config*.yml'
+        for module_conf in glob.glob(module_confs):
+
+            # Ignore reference confs
+            if REFERENCE_CONFIG_RE.match(module_conf):
+                continue
+
+            if os.path.isfile(module_conf) == False:
+                continue
+
+            module_file = header.format(module=module, beat_name=beat_name, docs_branch=docs_branch)
+            disabled_config_filename = os.path.basename(module_conf).replace('config', module) + '.disabled'
+
+            with open(module_conf) as f:
+                module_file += f.read()
+
+            # Write disabled module conf
+            with open(os.path.abspath('modules.d/') + '/' + disabled_config_filename, 'w') as f:
+                f.write(module_file)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Collects modules confs")
+    parser.add_argument("--beat", help="Beat name")
+    parser.add_argument("--docs_branch", help="Docs branch")
+
+    args = parser.parse_args()
+    beat_name = args.beat
+    docs_branch = args.docs_branch
+
+    collect(beat_name, docs_branch)