ECS Templates: stack alignment (#1062)

* ecs: support stack-aligned generated templates

* ecs: ship latest 1.x for ES 7+, keep ES6 locked to 1.10

* fix SECURE_INTEGRATION specs to properly detect ES version

* specs: force legacy specs to run with ecs disabled

* changelog enrtry and version bump

CHANGELOG.md

@@ -1,3 +1,8 @@
+## 11.4.0
+ - Updates ECS templates [#1062](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1062)
+   - Updates v1 templates to 1.12.1 for use with Elasticsearch 7.x and 8.x
+   - Updates BETA preview of ECS v8 templates for Elasticsearch 7.x and 8.x
+
 ## 11.3.3
  - Feat: add support for 'traces' data stream type [#1057](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1057)
 

Rakefile

@@ -1,36 +1,56 @@
 require "logstash/devutils/rake"
 
-ECS_VERSIONS = {
-    v1: 'v1.10.0', # WARNING: v1.11 breaks 6.x (see: https://github.com/elastic/ecs/issues/1649)
+task :'vendor-ecs-schemata' do
+  download_ecs_schemata(:v1, elasticsearch_major: 6, ecs_release_tag: 'v1.10.0') # WARNING: v1.11 breaks 6.x (see: https://github.com/elastic/ecs/issues/1649)
+  download_ecs_schemata(:v1, elasticsearch_major: 7, ecs_release_tag: 'v1.12.1')
+  download_ecs_schemata(:v1, elasticsearch_major: 8, ecs_release_tag: 'v1.12.1', generated_for: 7)
+
+  # PRERELEASE: 8.0 branch
+  # when pinning to released tag, remove BETA warning.
+  download_ecs_schemata(:v8, elasticsearch_major: 7, ecs_release_tag: '8.0')
+  download_ecs_schemata(:v8, elasticsearch_major: 8, ecs_release_tag: '8.0')
+end
+
+task :vendor => :'vendor-ecs-schemata'
 
-    # PRERELEASE: 8.0@{2021-11-02T20:09:42Z}
-    # when pinning to released tag, remove BETA warning.
-    v8: 'b3dace43afb0dce743605cdd4cc067a3a6b8131c',
-}
 
 ECS_LOGSTASH_INDEX_PATTERNS = %w(
   ecs-logstash-*
-)
+).freeze
 
-task :'vendor-ecs-schemata' do
-  download_ecs_schema(:v1, 6)
-  download_ecs_schema(:v1, 7)
-  download_ecs_schema(:v1, 8, 7) { |template| transform_for_es8!(template) }
+def download_ecs_schemata(ecs_major, elasticsearch_major:, ecs_release_tag:, generated_for: elasticsearch_major)
+  # when talking with ES >= 8, this plugin uses the v2 _index_template API and needs
+  # the generated monolith legacy index template to be transformed into a v2 index template
+  transform = Proc.new { |template| transform_legacy_template_to_v2!(template) if elasticsearch_major >= 8 }
+
+  return download_ecs_v1(elasticsearch_major: elasticsearch_major, ecs_release_tag: ecs_release_tag, generated_for: generated_for, &transform) if ecs_major == :v1
+
+  fail(ArgumentError, "Stack-aligned #{ecs_major} does not support `generated_for`") if generated_for != elasticsearch_major
 
-  download_ecs_schema(:v8, 7)
-  download_ecs_schema(:v8, 8, 7) { |template| transform_for_es8!(template) }
+  download_ecs_aligned(ecs_major, elasticsearch_major: elasticsearch_major, ecs_release_tag: ecs_release_tag, &transform)
+end
+
+def download_ecs_v1(elasticsearch_major:, ecs_release_tag:, generated_for: elasticsearch_major, &transform)
+  $stderr.puts("Vendoring v1 ECS template (#{ecs_release_tag}) for Elasticsearch #{elasticsearch_major}"+(elasticsearch_major==generated_for ? '': " (transformed from templates pre-generated for ES #{generated_for})"))
+
+  source_url = "/elastic/ecs/#{ecs_release_tag}/generated/elasticsearch/#{generated_for}/template.json"
+  download_and_transform(source_url: source_url, ecs_major: :v1, es_major: elasticsearch_major, &transform)
+end
+
+def download_ecs_aligned(ecs_major, elasticsearch_major:, ecs_release_tag:, &transform)
+  $stderr.puts("Vendoring Stack-aligned ECS template (#{ecs_release_tag}) for Elasticsearch #{elasticsearch_major}")
+
+  source_url = "/elastic/ecs/#{ecs_release_tag}/generated/elasticsearch/legacy/template.json"
+  download_and_transform(source_url: source_url, ecs_major: ecs_major, es_major: elasticsearch_major, &transform)
 end
-task :vendor => :'vendor-ecs-schemata'
 
-def download_ecs_schema(ecs_major_version, es_major, generated_for_es_major=es_major)
-  $stderr.puts("Vendoring ECS #{ecs_major_version} template for Elasticsearch #{es_major}"+(es_major==generated_for_es_major ? '': " (transformed from templates pre-generated for ES #{generated_for_es_major})"))
+def download_and_transform(source_url:, ecs_major:, es_major:)
   require 'net/http'
   require 'json'
   Net::HTTP.start('raw.githubusercontent.com', :use_ssl => true) do |http|
-    ecs_release_tag = ECS_VERSIONS.fetch(ecs_major_version)
-    response = http.get("/elastic/ecs/#{ecs_release_tag}/generated/elasticsearch/#{generated_for_es_major}/template.json")
+    response = http.get(source_url)
     fail "#{response.code} #{response.message}" unless (200...300).cover?(response.code.to_i)
-    template_directory = File.expand_path("../lib/logstash/outputs/elasticsearch/templates/ecs-#{ecs_major_version}", __FILE__)
+    template_directory = File.expand_path("../lib/logstash/outputs/elasticsearch/templates/ecs-#{ecs_major}", __FILE__)
     Dir.mkdir(template_directory) unless File.exists?(template_directory)
     File.open(File.join(template_directory, "/elasticsearch-#{es_major}x.json"), "w") do |handle|
       template = JSON.load(response.body)
@@ -46,8 +66,8 @@ def replace_index_patterns!(template, replacement_index_patterns)
   template.update('index_patterns' => replacement_index_patterns)
 end
 
-# destructively transforms a legacy template into an ES8-compatible index_template.
-def transform_for_es8!(template)
+# destructively transforms an ES7-style legacy template into an ES8-compatible index_template.
+def transform_legacy_template_to_v2!(template)
   # `settings` and `mappings` are now nested under top-level `template`
   template["template"] = {
     "settings" => template.delete("settings"),

logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.3.3'
+  s.version         = '11.4.0'
 
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"

spec/es_spec_helper.rb

@@ -59,7 +59,11 @@ def routing_field_name
   end
 
   def self.es_version
-    RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
+    [
+      nilify(RSpec.configuration.filter[:es_version]),
+      nilify(ENV['ES_VERSION']),
+      nilify(ENV['ELASTIC_STACK_VERSION']),
+    ].compact.first
   end
 
   RSpec::Matchers.define :have_hits do |expected|

spec/integration/outputs/ilm_spec.rb

@@ -236,7 +236,8 @@
   let (:settings) {
     {
         "ilm_enabled" => ilm_enabled,
-        "hosts" => "#{get_host_port()}"
+        "hosts" => "#{get_host_port()}",
+        "ecs_compatibility" => "disabled", # specs are tightly tied to non-ECS defaults
     }
   }
   let (:small_max_doc_policy) { max_docs_policy(3) }

spec/integration/outputs/ingest_pipeline_spec.rb

@@ -6,7 +6,8 @@
     settings = {
       "hosts" => "#{get_host_port()}",
       "pipeline" => "apache-logs",
-      "data_stream" => 'false'
+      "data_stream" => 'false',
+      "ecs_compatibility" => "disabled", # specs are tightly tied to non-ECS defaults
     }
     next LogStash::Outputs::ElasticSearch.new(settings)
   end

spec/integration/outputs/retry_spec.rb

@@ -45,7 +45,8 @@ def mock_actions_with_response(*resp)
       "template_overwrite" => true,
       "hosts" => get_host_port(),
       "retry_max_interval" => 64,
-      "retry_initial_interval" => 2
+      "retry_initial_interval" => 2,
+      "ecs_compatibility" => "disabled", # specs are tightly tied to non-ECS defaults
     }
     next LogStash::Outputs::ElasticSearch.new(settings)
   end