ecs: support stack-aligned generated templates

Rakefile

@@ -1,36 +1,53 @@
 require "logstash/devutils/rake"
 
-ECS_VERSIONS = {
-    v1: 'v1.10.0', # WARNING: v1.11 breaks 6.x (see: https://github.com/elastic/ecs/issues/1649)
+task :'vendor-ecs-schemata' do
+  # WARNING: v1.11 breaks 6.x (see: https://github.com/elastic/ecs/issues/1649)
+  download_ecs_schemata(:v1, elasticsearch_major: 6, ecs_release_tag: 'v1.10.0')
+  download_ecs_schemata(:v1, elasticsearch_major: 7, ecs_release_tag: 'v1.10.0')
+  download_ecs_schemata(:v1, elasticsearch_major: 8, ecs_release_tag: 'v1.10.0', generated_for: 7) { |template| transform_for_es8!(template) }
+
+  # PRERELEASE: 8.0 branch
+  # when pinning to released tag, remove BETA warning.
+  download_ecs_schemata(:v8, elasticsearch_major: 7, ecs_release_tag: '8.0') { |template| transform_for_es7!(template) if template.include?("template") }
+  download_ecs_schemata(:v8, elasticsearch_major: 8, ecs_release_tag: '8.0') { |template| transform_for_es8!(template) unless template.include?("template") }
+end
+
+task :vendor => :'vendor-ecs-schemata'
 
-    # PRERELEASE: 8.0@{2021-11-02T20:09:42Z}
-    # when pinning to released tag, remove BETA warning.
-    v8: 'b3dace43afb0dce743605cdd4cc067a3a6b8131c',
-}
 
 ECS_LOGSTASH_INDEX_PATTERNS = %w(
   ecs-logstash-*
-)
+).freeze
 
-task :'vendor-ecs-schemata' do
-  download_ecs_schema(:v1, 6)
-  download_ecs_schema(:v1, 7)
-  download_ecs_schema(:v1, 8, 7) { |template| transform_for_es8!(template) }
+def download_ecs_schemata(ecs_major, elasticsearch_major:, ecs_release_tag:, generated_for: elasticsearch_major, &transform)
+  return download_ecs_v1(elasticsearch_major: elasticsearch_major, ecs_release_tag: ecs_release_tag, generated_for: generated_for, &transform) if ecs_major == :v1
 
-  download_ecs_schema(:v8, 7)
-  download_ecs_schema(:v8, 8, 7) { |template| transform_for_es8!(template) }
+  fail(ArgumentError, "Stack-aligned #{ecs_major} does not support `generated_for`") if generated_for != elasticsearch_major
+
+  download_ecs_aligned(ecs_major, elasticsearch_major: elasticsearch_major, ecs_release_tag: ecs_release_tag, &transform)
+end
+
+def download_ecs_v1(elasticsearch_major:, ecs_release_tag:, generated_for: elasticsearch_major, &transform)
+  $stderr.puts("Vendoring v1 ECS template (#{ecs_release_tag}) for Elasticsearch #{elasticsearch_major}"+(elasticsearch_major==generated_for ? '': " (transformed from templates pre-generated for ES #{generated_for})"))
+
+  source_url = "/elastic/ecs/#{ecs_release_tag}/generated/elasticsearch/#{generated_for}/template.json"
+  download_and_transform(source_url: source_url, ecs_major: :v1, es_major: elasticsearch_major, &transform)
 end
-task :vendor => :'vendor-ecs-schemata'
 
-def download_ecs_schema(ecs_major_version, es_major, generated_for_es_major=es_major)
-  $stderr.puts("Vendoring ECS #{ecs_major_version} template for Elasticsearch #{es_major}"+(es_major==generated_for_es_major ? '': " (transformed from templates pre-generated for ES #{generated_for_es_major})"))
+def download_ecs_aligned(ecs_major, elasticsearch_major:, ecs_release_tag:, &transform)
+  $stderr.puts("Vendoring Stack-aligned ECS template (#{ecs_release_tag}) for Elasticsearch #{elasticsearch_major}")
+
+  source_url = "/elastic/ecs/#{ecs_release_tag}/generated/elasticsearch/legacy/template.json"
+  download_and_transform(source_url: source_url, ecs_major: ecs_major, es_major: elasticsearch_major, &transform)
+end
+
+def download_and_transform(source_url:, ecs_major:, es_major:)
   require 'net/http'
   require 'json'
   Net::HTTP.start('raw.githubusercontent.com', :use_ssl => true) do |http|
-    ecs_release_tag = ECS_VERSIONS.fetch(ecs_major_version)
-    response = http.get("/elastic/ecs/#{ecs_release_tag}/generated/elasticsearch/#{generated_for_es_major}/template.json")
+    response = http.get(source_url)
     fail "#{response.code} #{response.message}" unless (200...300).cover?(response.code.to_i)
-    template_directory = File.expand_path("../lib/logstash/outputs/elasticsearch/templates/ecs-#{ecs_major_version}", __FILE__)
+    template_directory = File.expand_path("../lib/logstash/outputs/elasticsearch/templates/ecs-#{ecs_major}", __FILE__)
     Dir.mkdir(template_directory) unless File.exists?(template_directory)
     File.open(File.join(template_directory, "/elasticsearch-#{es_major}x.json"), "w") do |handle|
       template = JSON.load(response.body)
@@ -46,7 +63,7 @@ def replace_index_patterns!(template, replacement_index_patterns)
   template.update('index_patterns' => replacement_index_patterns)
 end
 
-# destructively transforms a legacy template into an ES8-compatible index_template.
+# destructively transforms an ES7-style legacy template into an ES8-compatible index_template.
 def transform_for_es8!(template)
   # `settings` and `mappings` are now nested under top-level `template`
   template["template"] = {
@@ -63,3 +80,20 @@ def transform_for_es8!(template)
 
   nil
 end
+
+# destructively transforms an ES8-style legacy index_template into an ES7-compatible legacy template.
+def transform_for_es7!(template)
+  # `settings` and `mappings` in source now nested under top-level `template`,
+  # need to be top-level in es7 template
+  inner = template.delete("template")
+  template.update(inner)
+
+  # `order` is gone, replaced with `priority`.
+  template.delete('priority')
+  template["order"] = 2
+
+  # a new free-form `_meta` exists for ES8+, but isn't backward-supported. Remove it.
+  template.delete('_meta')
+
+  nil
+end