From 1a9de7a502dd12db23975ae27566c331bb5845df Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Wed, 22 Dec 2021 21:41:31 +0000 Subject: [PATCH 1/5] ecs: support stack-aligned generated templates --- Rakefile | 63 +++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 21 deletions(-) diff --git a/Rakefile b/Rakefile index 4115a41f..4ef4ab63 100644 --- a/Rakefile +++ b/Rakefile @@ -1,36 +1,57 @@ require "logstash/devutils/rake" -ECS_VERSIONS = { - v1: 'v1.10.0', # WARNING: v1.11 breaks 6.x (see: https://github.com/elastic/ecs/issues/1649) +task :'vendor-ecs-schemata' do + # WARNING: v1.11 breaks 6.x (see: https://github.com/elastic/ecs/issues/1649) + download_ecs_schemata(:v1, elasticsearch_major: 6, ecs_release_tag: 'v1.10.0') + download_ecs_schemata(:v1, elasticsearch_major: 7, ecs_release_tag: 'v1.10.0') + download_ecs_schemata(:v1, elasticsearch_major: 8, ecs_release_tag: 'v1.10.0', generated_for: 7) + + # PRERELEASE: 8.0 branch + # when pinning to released tag, remove BETA warning. + download_ecs_schemata(:v8, elasticsearch_major: 7, ecs_release_tag: '8.0') + download_ecs_schemata(:v8, elasticsearch_major: 8, ecs_release_tag: '8.0') +end + +task :vendor => :'vendor-ecs-schemata' - # PRERELEASE: 8.0@{2021-11-02T20:09:42Z} - # when pinning to released tag, remove BETA warning. - v8: 'b3dace43afb0dce743605cdd4cc067a3a6b8131c', -} ECS_LOGSTASH_INDEX_PATTERNS = %w( ecs-logstash-* -) +).freeze -task :'vendor-ecs-schemata' do - download_ecs_schema(:v1, 6) - download_ecs_schema(:v1, 7) - download_ecs_schema(:v1, 8, 7) { |template| transform_for_es8!(template) } +def download_ecs_schemata(ecs_major, elasticsearch_major:, ecs_release_tag:, generated_for: elasticsearch_major) + # when talking with ES >= 8, this plugin uses the v2 _index_template API and needs + # the generated monolith legacy index template to be transformed into a v2 index template + transform = Proc.new { |template| transform_legacy_template_to_v2!(template) if elasticsearch_major >= 8 } + + return download_ecs_v1(elasticsearch_major: elasticsearch_major, ecs_release_tag: ecs_release_tag, generated_for: generated_for, &transform) if ecs_major == :v1 + + fail(ArgumentError, "Stack-aligned #{ecs_major} does not support `generated_for`") if generated_for != elasticsearch_major - download_ecs_schema(:v8, 7) - download_ecs_schema(:v8, 8, 7) { |template| transform_for_es8!(template) } + download_ecs_aligned(ecs_major, elasticsearch_major: elasticsearch_major, ecs_release_tag: ecs_release_tag, &transform) +end + +def download_ecs_v1(elasticsearch_major:, ecs_release_tag:, generated_for: elasticsearch_major, &transform) + $stderr.puts("Vendoring v1 ECS template (#{ecs_release_tag}) for Elasticsearch #{elasticsearch_major}"+(elasticsearch_major==generated_for ? '': " (transformed from templates pre-generated for ES #{generated_for})")) + + source_url = "/elastic/ecs/#{ecs_release_tag}/generated/elasticsearch/#{generated_for}/template.json" + download_and_transform(source_url: source_url, ecs_major: :v1, es_major: elasticsearch_major, &transform) +end + +def download_ecs_aligned(ecs_major, elasticsearch_major:, ecs_release_tag:, &transform) + $stderr.puts("Vendoring Stack-aligned ECS template (#{ecs_release_tag}) for Elasticsearch #{elasticsearch_major}") + + source_url = "/elastic/ecs/#{ecs_release_tag}/generated/elasticsearch/legacy/template.json" + download_and_transform(source_url: source_url, ecs_major: ecs_major, es_major: elasticsearch_major, &transform) end -task :vendor => :'vendor-ecs-schemata' -def download_ecs_schema(ecs_major_version, es_major, generated_for_es_major=es_major) - $stderr.puts("Vendoring ECS #{ecs_major_version} template for Elasticsearch #{es_major}"+(es_major==generated_for_es_major ? '': " (transformed from templates pre-generated for ES #{generated_for_es_major})")) +def download_and_transform(source_url:, ecs_major:, es_major:) require 'net/http' require 'json' Net::HTTP.start('raw.githubusercontent.com', :use_ssl => true) do |http| - ecs_release_tag = ECS_VERSIONS.fetch(ecs_major_version) - response = http.get("/elastic/ecs/#{ecs_release_tag}/generated/elasticsearch/#{generated_for_es_major}/template.json") + response = http.get(source_url) fail "#{response.code} #{response.message}" unless (200...300).cover?(response.code.to_i) - template_directory = File.expand_path("../lib/logstash/outputs/elasticsearch/templates/ecs-#{ecs_major_version}", __FILE__) + template_directory = File.expand_path("../lib/logstash/outputs/elasticsearch/templates/ecs-#{ecs_major}", __FILE__) Dir.mkdir(template_directory) unless File.exists?(template_directory) File.open(File.join(template_directory, "/elasticsearch-#{es_major}x.json"), "w") do |handle| template = JSON.load(response.body) @@ -46,8 +67,8 @@ def replace_index_patterns!(template, replacement_index_patterns) template.update('index_patterns' => replacement_index_patterns) end -# destructively transforms a legacy template into an ES8-compatible index_template. -def transform_for_es8!(template) +# destructively transforms an ES7-style legacy template into an ES8-compatible index_template. +def transform_legacy_template_to_v2!(template) # `settings` and `mappings` are now nested under top-level `template` template["template"] = { "settings" => template.delete("settings"), From 84a0e8a2eec9db264687f9341091c1a83c7c980e Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Wed, 22 Dec 2021 21:43:48 +0000 Subject: [PATCH 2/5] ecs: ship latest 1.x for ES 7+, keep ES6 locked to 1.10 --- Rakefile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Rakefile b/Rakefile index 4ef4ab63..8c18993c 100644 --- a/Rakefile +++ b/Rakefile @@ -1,10 +1,9 @@ require "logstash/devutils/rake" task :'vendor-ecs-schemata' do - # WARNING: v1.11 breaks 6.x (see: https://github.com/elastic/ecs/issues/1649) - download_ecs_schemata(:v1, elasticsearch_major: 6, ecs_release_tag: 'v1.10.0') - download_ecs_schemata(:v1, elasticsearch_major: 7, ecs_release_tag: 'v1.10.0') - download_ecs_schemata(:v1, elasticsearch_major: 8, ecs_release_tag: 'v1.10.0', generated_for: 7) + download_ecs_schemata(:v1, elasticsearch_major: 6, ecs_release_tag: 'v1.10.0') # WARNING: v1.11 breaks 6.x (see: https://github.com/elastic/ecs/issues/1649) + download_ecs_schemata(:v1, elasticsearch_major: 7, ecs_release_tag: 'v1.12.1') + download_ecs_schemata(:v1, elasticsearch_major: 8, ecs_release_tag: 'v1.12.1', generated_for: 7) # PRERELEASE: 8.0 branch # when pinning to released tag, remove BETA warning. From 68d545127abb78b5b1d89e2dd98a2641325e0e73 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Wed, 12 Jan 2022 22:58:45 +0000 Subject: [PATCH 3/5] fix SECURE_INTEGRATION specs to properly detect ES version --- spec/es_spec_helper.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/spec/es_spec_helper.rb b/spec/es_spec_helper.rb index c0b44b16..8ea301e0 100644 --- a/spec/es_spec_helper.rb +++ b/spec/es_spec_helper.rb @@ -59,7 +59,11 @@ def routing_field_name end def self.es_version - RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION'] + [ + nilify(RSpec.configuration.filter[:es_version]), + nilify(ENV['ES_VERSION']), + nilify(ENV['ELASTIC_STACK_VERSION']), + ].compact.first end RSpec::Matchers.define :have_hits do |expected| From 315c0d56b1e7e51300a1ed9aa3f1b9b8f85d9ddd Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Wed, 12 Jan 2022 23:02:01 +0000 Subject: [PATCH 4/5] specs: force legacy specs to run with ecs disabled --- spec/integration/outputs/ilm_spec.rb | 3 ++- spec/integration/outputs/ingest_pipeline_spec.rb | 3 ++- spec/integration/outputs/retry_spec.rb | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/spec/integration/outputs/ilm_spec.rb b/spec/integration/outputs/ilm_spec.rb index d3755337..e6c2dce1 100644 --- a/spec/integration/outputs/ilm_spec.rb +++ b/spec/integration/outputs/ilm_spec.rb @@ -236,7 +236,8 @@ let (:settings) { { "ilm_enabled" => ilm_enabled, - "hosts" => "#{get_host_port()}" + "hosts" => "#{get_host_port()}", + "ecs_compatibility" => "disabled", # specs are tightly tied to non-ECS defaults } } let (:small_max_doc_policy) { max_docs_policy(3) } diff --git a/spec/integration/outputs/ingest_pipeline_spec.rb b/spec/integration/outputs/ingest_pipeline_spec.rb index c76fdaef..c550f9ab 100644 --- a/spec/integration/outputs/ingest_pipeline_spec.rb +++ b/spec/integration/outputs/ingest_pipeline_spec.rb @@ -6,7 +6,8 @@ settings = { "hosts" => "#{get_host_port()}", "pipeline" => "apache-logs", - "data_stream" => 'false' + "data_stream" => 'false', + "ecs_compatibility" => "disabled", # specs are tightly tied to non-ECS defaults } next LogStash::Outputs::ElasticSearch.new(settings) end diff --git a/spec/integration/outputs/retry_spec.rb b/spec/integration/outputs/retry_spec.rb index 699ffeb6..5493f44c 100644 --- a/spec/integration/outputs/retry_spec.rb +++ b/spec/integration/outputs/retry_spec.rb @@ -45,7 +45,8 @@ def mock_actions_with_response(*resp) "template_overwrite" => true, "hosts" => get_host_port(), "retry_max_interval" => 64, - "retry_initial_interval" => 2 + "retry_initial_interval" => 2, + "ecs_compatibility" => "disabled", # specs are tightly tied to non-ECS defaults } next LogStash::Outputs::ElasticSearch.new(settings) end From c05fefb2546eb831eb073708b0475412a8fd9daf Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Wed, 12 Jan 2022 23:26:34 +0000 Subject: [PATCH 5/5] changelog enrtry and version bump --- CHANGELOG.md | 5 +++++ logstash-output-elasticsearch.gemspec | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38d5149c..abec0658 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 11.4.0 + - Updates ECS templates [#1062](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1062) + - Updates v1 templates to 1.12.1 for use with Elasticsearch 7.x and 8.x + - Updates BETA preview of ECS v8 templates for Elasticsearch 7.x and 8.x + ## 11.3.3 - Feat: add support for 'traces' data stream type [#1057](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1057) diff --git a/logstash-output-elasticsearch.gemspec b/logstash-output-elasticsearch.gemspec index 4ce99ffa..e8ccad67 100644 --- a/logstash-output-elasticsearch.gemspec +++ b/logstash-output-elasticsearch.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'logstash-output-elasticsearch' - s.version = '11.3.3' + s.version = '11.4.0' s.licenses = ['apache-2.0'] s.summary = "Stores logs in Elasticsearch"