Refactor index management (#10347)

Changes: - introduce Index + ILM manager interfaces responsible for: - create ILM policy - create write alias - create and load template - separate interfaces for re-use: manager, apihandler - provide default API handler for elasticsearch.Client - Index + ILM support interfaces, that can be swapped out with custom implementation (still WIP) - Beat implementing custom managers have to decide on config file format - ILM settings have been moved to `ilm.setup` - Introduce `setup.ilm.enabled: auto` (new default) - Index manager will detect on connect to Elasticsearch if ILM can be enabled or not - Lazily chooses write alias or old-style index for publishing - Index Manager does not check ILM availabilty on startup, but on first connect (so to not block beats startup) - If Elasticsearch < 7.0, then `auto` defaults to false - If Elasticsearch > 7.0, then `auto` checks if ILM is available and enbaled - If `setup.ilm.enabled: true`, then fail check if version does not support ILM or ILM is disabled - Publisher pipeline does not use output package anymore to create an output, but a callback -> Beat instance itself creates output instances, passing IndexManager and other info to the output loader. - Outputs can use IndexManager in order to create the per event index selector - remove fragile manipulation of `common.Config` objects, that have been already partially parsed. - Ensure ILM can be setup if `setup.ilm.enabled: true`, but elasticsearch output is configured via Central Management Followup: update docs
elastic · Jan 31, 2019 · 2bbf828 · 2bbf828
1 parent ed2c8d5
commit 2bbf828
Show file tree

Hide file tree

Showing 82 changed files with 2,901 additions and 691 deletions.
diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc
@@ -19,6 +19,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2..master[Check the HEAD di
 The list below covers the major changes between 7.0.0-alpha2 and master only.
 
 ==== Breaking changes
+- Outputs receive Index Manager as additional parameter. The index manager can
+  be used to create an index selector. {pull}10347[10347]
 
 ==== Bugfixes
 
@@ -27,3 +29,5 @@ The list below covers the major changes between 7.0.0-alpha2 and master only.
 - Allow multiple object type configurations per field. {pull}9772[9772]
 - Move agent metadata addition to a processor. {pull}9952[9952]
 - Add (*common.Config).Has and (*common.Config).Remove. {pull}10363[10363]
+- Introduce ILM and IndexManagment support to beat.Settings. {pull}10347[10347]
+- Introduce ILM and IndexManagement support to beat.Settings. {pull}10347[10347]
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -26,6 +26,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove --setup command line flag. {pull}10138[10138]
 - Remove --version command line flag. {pull}10138[10138]
 - Remove --configtest command line flag. {pull}10138[10138]
+- Move output.elasticsearch.ilm settings to setup.ilm. {pull}10347[10347]
+- ILM will be available by default if Elasticsearch > 7.0 is used. {pull}10347[10347]
 
 *Auditbeat*
 
@@ -194,6 +196,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `add_fields` processor. {pull}10119[10119]
 - Add Kibana field formatter to bytes fields. {pull}10184[10184]
 - Document a few more `auditd.log.*` fields. {pull}10192[10192]
+- Add ILM mode `auto` to setup.ilm.enabled setting. This new default value detects if ILM is available {pull}10347[10347]
+- Add support to read ILM policy from external JSON file. {pull}10347[10347]
+- Add `overwrite` and `check_exists` settings to ILM support. {pull}10347[10347]
 
 *Auditbeat*
 

diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml
@@ -358,11 +358,6 @@ output.elasticsearch:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200
   hosts: ["localhost:9200"]
 
-  # Enabled ilm (beta) to use index lifecycle management instead daily indices.
-  #ilm.enabled: false
-  #ilm.rollover_alias: "auditbeat"
-  #ilm.pattern: "{now/d}-000001"
-
   # Set gzip compression level.
   #compression_level: 0
 
@@ -1009,6 +1004,25 @@ setup.template.settings:
   #_source:
     #enabled: false
 
+#============================== Setup ILM =====================================
+
+# Configure Index Lifecycle Management Index Lifecycle Management creates a
+# write alias and adds additional settings to the template.
+# The elasticsearch.output.index setting will be replaced with the write alias
+# if ILM is enabled.
+
+# Enabled ILM support. Valid values are true, false, and auto. The beat will
+# detect availabilty of Index Lifecycle Management in Elasticsearch and enable
+# or disable ILM support.
+#setup.ilm.enabled: auto
+
+# Configure the ILM write alias name.
+#setup.ilm.rollover_alias: "auditbeat"
+
+# Configure rollover index pattern.
+#setup.ilm.pattern: "{now/d}-000001"
+
+
 #============================== Kibana =====================================
 
 # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.

diff --git a/auditbeat/auditbeat.yml b/auditbeat/auditbeat.yml
@@ -121,9 +121,6 @@ output.elasticsearch:
   # Array of hosts to connect to.
   hosts: ["localhost:9200"]
 
-  # Enabled ilm (beta) to use index lifecycle management instead daily indices.
-  #ilm.enabled: false
-
   # Optional protocol and basic auth credentials.
   #protocol: "https"
   #username: "elastic"

diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -1059,11 +1059,6 @@ output.elasticsearch:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200
   hosts: ["localhost:9200"]
 
-  # Enabled ilm (beta) to use index lifecycle management instead daily indices.
-  #ilm.enabled: false
-  #ilm.rollover_alias: "filebeat"
-  #ilm.pattern: "{now/d}-000001"
-
   # Set gzip compression level.
   #compression_level: 0
 
@@ -1710,6 +1705,25 @@ setup.template.settings:
   #_source:
     #enabled: false
 
+#============================== Setup ILM =====================================
+
+# Configure Index Lifecycle Management Index Lifecycle Management creates a
+# write alias and adds additional settings to the template.
+# The elasticsearch.output.index setting will be replaced with the write alias
+# if ILM is enabled.
+
+# Enabled ILM support. Valid values are true, false, and auto. The beat will
+# detect availabilty of Index Lifecycle Management in Elasticsearch and enable
+# or disable ILM support.
+#setup.ilm.enabled: auto
+
+# Configure the ILM write alias name.
+#setup.ilm.rollover_alias: "filebeat"
+
+# Configure rollover index pattern.
+#setup.ilm.pattern: "{now/d}-000001"
+
+
 #============================== Kibana =====================================
 
 # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.

diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml
@@ -149,9 +149,6 @@ output.elasticsearch:
   # Array of hosts to connect to.
   hosts: ["localhost:9200"]
 
-  # Enabled ilm (beta) to use index lifecycle management instead daily indices.
-  #ilm.enabled: false
-
   # Optional protocol and basic auth credentials.
   #protocol: "https"
   #username: "elastic"

diff --git a/filebeat/tests/system/config/filebeat.yml.j2 b/filebeat/tests/system/config/filebeat.yml.j2
@@ -105,6 +105,19 @@ filebeat.config.{{ reload_type|default("inputs") }}:
   {% endif -%}
 {% endif -%}
 
+{% if ilm %}
+setup.ilm:
+  enabled: {{ ilm.enabled | default("auto") }}
+  policy_name: libbeat-test-default-policy
+  {% if ilm.pattern %}
+  pattern: {{ ilm.pattern }}
+  {% endif %}
+  {% if ilm.rollover_alias %}
+  rollover_alias: {{ ilm.rollover_alias }}
+  {% endif %}
+{% endif %}
+
+
 #============================== Autodiscover ==================================
 
 {% if autodiscover %}

diff --git a/filebeat/tests/system/config/filebeat_inputs.yml.j2 b/filebeat/tests/system/config/filebeat_inputs.yml.j2
@@ -7,6 +7,19 @@ filebeat.inputs:
 {% endfor %}
 filebeat.registry_file: {{ beat.working_dir + '/' }}{{ registryFile|default("registry")}}
 
+{% if ilm %}
+setup.ilm:
+  enabled: {{ ilm.enabled | default("auto") }}
+  policy_name: libbeat-test-default-policy
+  {% if ilm.pattern %}
+  pattern: {{ ilm.pattern }}
+  {% endif %}
+  {% if ilm.rollover_alias %}
+  rollover_alias: {{ ilm.rollover_alias }}
+  {% endif %}
+{% endif %}
+
+
 output.file:
   path: {{ output_file_path|default(beat.working_dir + "/output") }}
   filename: "{{ output_file_filename|default("filebeat") }}"

diff --git a/filebeat/tests/system/config/filebeat_modules.yml.j2 b/filebeat/tests/system/config/filebeat_modules.yml.j2
@@ -15,3 +15,16 @@ setup.kibana.host: {{ kibana_url }}
 {% if kibana_path %}
 setup.dashboards.directory: {{ kibana_path }}
 {% endif %}
+
+{% if ilm %}
+setup.ilm:
+  enabled: {{ ilm.enabled | default("auto") }}
+  policy_name: libbeat-test-default-policy
+  {% if ilm.pattern %}
+  pattern: {{ ilm.pattern }}
+  {% endif %}
+  {% if ilm.rollover_alias %}
+  rollover_alias: {{ ilm.rollover_alias }}
+  {% endif %}
+{% endif %}
+
diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py
@@ -96,7 +96,7 @@ def test_fileset_file(self, module, fileset, test_file):
             template_name="filebeat_modules",
             output=cfgfile,
             index_name=self.index_name,
-            elasticsearch_url=self.elasticsearch_url
+            elasticsearch_url=self.elasticsearch_url,
         )
 
         self.run_on_file(
@@ -118,6 +118,7 @@ def run_on_file(self, module, fileset, test_file, cfgfile):
             self.filebeat, "-systemTest",
             "-e", "-d", "*", "-once",
             "-c", cfgfile,
+            "-E", "setup.ilm.enabled=false",
             "-modules={}".format(module),
             "-M", "{module}.*.enabled=false".format(module=module),
             "-M", "{module}.{fileset}.enabled=true".format(

diff --git a/filebeat/tests/system/test_pipeline.py b/filebeat/tests/system/test_pipeline.py
@@ -62,6 +62,7 @@ def test_input_pipeline_config(self):
             pipeline="test",
             setup_template_name=index_name,
             setup_template_pattern=index_name + "*",
+            ilm={"enabled": False},
         )
 
         os.mkdir(self.working_dir + "/log/")

diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml
@@ -502,11 +502,6 @@ output.elasticsearch:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200
   hosts: ["localhost:9200"]
 
-  # Enabled ilm (beta) to use index lifecycle management instead daily indices.
-  #ilm.enabled: false
-  #ilm.rollover_alias: "heartbeat"
-  #ilm.pattern: "{now/d}-000001"
-
   # Set gzip compression level.
   #compression_level: 0
 
@@ -1153,6 +1148,25 @@ setup.template.settings:
   #_source:
     #enabled: false
 
+#============================== Setup ILM =====================================
+
+# Configure Index Lifecycle Management Index Lifecycle Management creates a
+# write alias and adds additional settings to the template.
+# The elasticsearch.output.index setting will be replaced with the write alias
+# if ILM is enabled.
+
+# Enabled ILM support. Valid values are true, false, and auto. The beat will
+# detect availabilty of Index Lifecycle Management in Elasticsearch and enable
+# or disable ILM support.
+#setup.ilm.enabled: auto
+
+# Configure the ILM write alias name.
+#setup.ilm.rollover_alias: "heartbeat"
+
+# Configure rollover index pattern.
+#setup.ilm.pattern: "{now/d}-000001"
+
+
 #============================== Kibana =====================================
 
 # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.

diff --git a/heartbeat/heartbeat.yml b/heartbeat/heartbeat.yml
@@ -106,9 +106,6 @@ output.elasticsearch:
   # Array of hosts to connect to.
   hosts: ["localhost:9200"]
 
-  # Enabled ilm (beta) to use index lifecycle management instead daily indices.
-  #ilm.enabled: false
-
   # Optional protocol and basic auth credentials.
   #protocol: "https"
   #username: "elastic"

diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml
@@ -298,11 +298,6 @@ output.elasticsearch:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200
   hosts: ["localhost:9200"]
 
-  # Enabled ilm (beta) to use index lifecycle management instead daily indices.
-  #ilm.enabled: false
-  #ilm.rollover_alias: "journalbeat"
-  #ilm.pattern: "{now/d}-000001"
-
   # Set gzip compression level.
   #compression_level: 0
 
@@ -949,6 +944,25 @@ setup.template.settings:
   #_source:
     #enabled: false
 
+#============================== Setup ILM =====================================
+
+# Configure Index Lifecycle Management Index Lifecycle Management creates a
+# write alias and adds additional settings to the template.
+# The elasticsearch.output.index setting will be replaced with the write alias
+# if ILM is enabled.
+
+# Enabled ILM support. Valid values are true, false, and auto. The beat will
+# detect availabilty of Index Lifecycle Management in Elasticsearch and enable
+# or disable ILM support.
+#setup.ilm.enabled: auto
+
+# Configure the ILM write alias name.
+#setup.ilm.rollover_alias: "journalbeat"
+
+# Configure rollover index pattern.
+#setup.ilm.pattern: "{now/d}-000001"
+
+
 #============================== Kibana =====================================
 
 # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.

diff --git a/journalbeat/journalbeat.yml b/journalbeat/journalbeat.yml
@@ -118,9 +118,6 @@ output.elasticsearch:
   # Array of hosts to connect to.
   hosts: ["localhost:9200"]
 
-  # Enabled ilm (beta) to use index lifecycle management instead daily indices.
-  #ilm.enabled: false
-
   # Optional protocol and basic auth credentials.
   #protocol: "https"
   #username: "elastic"

diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml
@@ -246,11 +246,6 @@ output.elasticsearch:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200
   hosts: ["localhost:9200"]
 
-  # Enabled ilm (beta) to use index lifecycle management instead daily indices.
-  #ilm.enabled: false
-  #ilm.rollover_alias: "beat-index-prefix"
-  #ilm.pattern: "{now/d}-000001"
-
   # Set gzip compression level.
   #compression_level: 0
 
@@ -897,6 +892,25 @@ setup.template.settings:
   #_source:
     #enabled: false
 
+#============================== Setup ILM =====================================
+
+# Configure Index Lifecycle Management Index Lifecycle Management creates a
+# write alias and adds additional settings to the template.
+# The elasticsearch.output.index setting will be replaced with the write alias
+# if ILM is enabled.
+
+# Enabled ILM support. Valid values are true, false, and auto. The beat will
+# detect availabilty of Index Lifecycle Management in Elasticsearch and enable
+# or disable ILM support.
+#setup.ilm.enabled: auto
+
+# Configure the ILM write alias name.
+#setup.ilm.rollover_alias: "beat-index-prefix"
+
+# Configure rollover index pattern.
+#setup.ilm.pattern: "{now/d}-000001"
+
+
 #============================== Kibana =====================================
 
 # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.

diff --git a/libbeat/_meta/config.yml b/libbeat/_meta/config.yml
@@ -66,9 +66,6 @@ output.elasticsearch:
   # Array of hosts to connect to.
   hosts: ["localhost:9200"]
 
-  # Enabled ilm (beta) to use index lifecycle management instead daily indices.
-  #ilm.enabled: false
-
   # Optional protocol and basic auth credentials.
   #protocol: "https"
   #username: "elastic"

diff --git a/libbeat/cmd/export.go b/libbeat/cmd/export.go
@@ -33,7 +33,7 @@ func genExportCmd(settings instance.Settings, name, idxPrefix, beatVersion strin
 	exportCmd.AddCommand(export.GenExportConfigCmd(settings, name, idxPrefix, beatVersion))
 	exportCmd.AddCommand(export.GenTemplateConfigCmd(settings, name, idxPrefix, beatVersion))
 	exportCmd.AddCommand(export.GenDashboardCmd(name, idxPrefix, beatVersion))
-	exportCmd.AddCommand(export.GenGetILMPolicyCmd())
+	exportCmd.AddCommand(export.GenGetILMPolicyCmd(settings, name, idxPrefix, beatVersion))
 
 	return exportCmd
 }