From f9150275a55dc5a8416438ec33645d25d3da639b Mon Sep 17 00:00:00 2001 From: beats-jenkins Date: Fri, 1 Feb 2019 10:13:42 +0100 Subject: [PATCH 1/2] Disable migration aliases in index pattern The migration aliases should not show up in the index pattern if `migration.enabled: false`. For this to happen, the Kibana index pattern must be generated on the fly instead of packaging it with each Beat. This PR introduces the generation of the index pattern when Kibana data is loaded. APM still needs the index pattern as file. For this the export command `index-pattern` was added. It will print the index pattern to the standard out: ``` ./metricbeat export index-pattern > pattern.json ``` The commands to generate the index pattern in the dev environment were removed. For checking if aliases are supported, the Kibana version is checked. Fully accurate would be to check the Elasticsearch version as it depends on the ES version in the end and not Kibana. But it's assume that in general the same minor version is used. The reason not Elasticsearch is checked as it would potentially require additional config options and adds unnecessary complexity. For the index pattern the internal fields.go are used. Even if fields.yml is configured still fields.go is used. This is the same behavior as we had so far when the index pattern was generated. It could be improved in the future to also support a fields.yml for the generation if needed. In general this PR tried to change as little code as possible. The code and tests around the Kibana dashboard generation and index pattern generation is not very nice. One reason is that it also contains old logic which was used for previous versions but also has the ability to read dashboards from a zip file. Because of this all the old capabilities have to stay in the code for now. The code should be cleaned up at a later stage. Further changes: * Added system tests to Filebeat to check for correct content in index pattern when migration is enabled. * Fix double generation of common.yml in Metricbeat. It seems some changes in the past caused that the common.yml file was contained in two fields.go files in Metricbeat --- CHANGELOG-developer.next.asciidoc | 2 + CHANGELOG.next.asciidoc | 1 + .../kibana_index_pattern.go | 92 ------------- dev-tools/mage/kibana.go | 17 +-- filebeat/tests/system/test_index_pattern.py | 32 +++++ libbeat/cmd/export.go | 1 + libbeat/cmd/export/index_pattern.go | 87 +++++++++++++ libbeat/cmd/instance/beat.go | 40 +++++- libbeat/common/field.go | 19 ++- libbeat/dashboards/dashboards.go | 23 ++-- libbeat/dashboards/importer.go | 20 +-- libbeat/dashboards/kibana_loader.go | 17 ++- libbeat/kibana/fields_transformer.go | 8 +- libbeat/kibana/fields_transformer_test.go | 23 ++-- libbeat/kibana/index_pattern_generator.go | 45 +++---- .../kibana/index_pattern_generator_test.go | 121 +++++++----------- libbeat/kibana/transformer.go | 4 +- libbeat/scripts/Makefile | 3 - libbeat/tests/system/test_index_pattern.py | 21 +++ metricbeat/Makefile | 2 +- metricbeat/include/fields.go | 36 ------ 21 files changed, 315 insertions(+), 299 deletions(-) delete mode 100644 dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go create mode 100644 filebeat/tests/system/test_index_pattern.py create mode 100644 libbeat/cmd/export/index_pattern.go create mode 100644 libbeat/tests/system/test_index_pattern.py delete mode 100644 metricbeat/include/fields.go diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index eca8bfdc6a1..530b638a704 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -21,6 +21,7 @@ The list below covers the major changes between 7.0.0-alpha2 and master only. ==== Breaking changes - Outputs receive Index Manager as additional parameter. The index manager can be used to create an index selector. {pull}10347[10347] +- Remove support for loading dashboards to Elasticsearch 5. {pull}10451[10451] ==== Bugfixes @@ -31,3 +32,4 @@ The list below covers the major changes between 7.0.0-alpha2 and master only. - Add (*common.Config).Has and (*common.Config).Remove. {pull}10363[10363] - Introduce ILM and IndexManagment support to beat.Settings. {pull}10347[10347] - Introduce ILM and IndexManagement support to beat.Settings. {pull}10347[10347] +- Generating index pattern on demand instead of shipping them in the packages. {pull}10478[10478] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 46a95df13b4..daeef17620a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -203,6 +203,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add ILM mode `auto` to setup.ilm.enabled setting. This new default value detects if ILM is available {pull}10347[10347] - Add support to read ILM policy from external JSON file. {pull}10347[10347] - Add `overwrite` and `check_exists` settings to ILM support. {pull}10347[10347] +- Generate Kibana index pattern on demand instead of using a local file. {pull}10478[10478] *Auditbeat* diff --git a/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go b/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go deleted file mode 100644 index 4be7a8d808c..00000000000 --- a/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go +++ /dev/null @@ -1,92 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package main - -import ( - "flag" - "log" - "path/filepath" - - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/kibana" - "github.com/elastic/beats/libbeat/version" -) - -var usageText = ` -Usage: kibana_index_pattern [flags] - kibana_index_pattern generates Kibana index patterns from the Beat's - fields.yml file. It will create a index pattern file that is usable with both - Kibana 6.x and 7.x. -Options: -`[1:] - -var ( - beatName string - beatVersion string - indexPattern string - fieldsYAMLFile string - outputDir string -) - -func init() { - flag.StringVar(&beatName, "beat", "", "Name of the beat. (Required)") - flag.StringVar(&beatVersion, "version", version.GetDefaultVersion(), "Beat version. (Required)") - flag.StringVar(&indexPattern, "index", "", "Kibana index pattern. (Required)") - flag.StringVar(&fieldsYAMLFile, "fields", "fields.yml", "fields.yml file containing all fields used by the Beat.") - flag.StringVar(&outputDir, "out", "build/kibana", "Output dir.") -} - -func main() { - log.SetFlags(0) - flag.Parse() - - if beatName == "" { - log.Fatal("Name of the Beat must be set (-beat).") - } - - if beatVersion == "" { - log.Fatal("Beat version must be set (-version).") - } - - if indexPattern == "" { - log.Fatal("Index pattern must be set (-index).") - } - - versions := []string{ - "6.0.0", - } - for _, version := range versions { - version, _ := common.NewVersion(version) - indexPattern, err := kibana.NewGenerator(indexPattern, beatName, fieldsYAMLFile, outputDir, beatVersion, *version) - if err != nil { - log.Fatal(err) - } - - file, err := indexPattern.Generate() - if err != nil { - log.Fatalf("ERROR: %s", err) - } - - // Log output file location. - absFile, err := filepath.Abs(file) - if err != nil { - absFile = file - } - log.Printf(">> The index pattern was created under %v", absFile) - } -} diff --git a/dev-tools/mage/kibana.go b/dev-tools/mage/kibana.go index 1b1b7ecc80e..1240d1ca562 100644 --- a/dev-tools/mage/kibana.go +++ b/dev-tools/mage/kibana.go @@ -74,22 +74,7 @@ func KibanaDashboards(moduleDirs ...string) error { return err } - beatVersion, err := BeatQualifiedVersion() - if err != nil { - return err - } - - // Generate Kibana index pattern files from fields.yml. - indexPatternCmd := sh.RunCmd("go", "run", - filepath.Join(esBeatsDir, "dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go"), - "-beat", BeatName, - "-version", beatVersion, - "-index", BeatIndexPrefix+"-*", - "-fields", "fields.yml", - "-out", kibanaBuildDir, - ) - - return indexPatternCmd() + return nil } // PackageKibanaDashboardsFromBuildDir reconfigures the packaging configuration diff --git a/filebeat/tests/system/test_index_pattern.py b/filebeat/tests/system/test_index_pattern.py new file mode 100644 index 00000000000..69c0d43d20a --- /dev/null +++ b/filebeat/tests/system/test_index_pattern.py @@ -0,0 +1,32 @@ +import os +import unittest +from filebeat import BaseTest + + +class Test(BaseTest): + + def test_export_index_pattern(self): + """ + Test export index pattern + """ + self.render_config_template() + exit_code = self.run_beat( + logging_args=[], + extra_args=["export", "index-pattern"]) + + assert exit_code == 0 + assert self.log_contains('"objects": [') + assert self.log_contains('beat.name') == False + + def test_export_index_pattern_migration(self): + """ + Test export index pattern with migration flag enabled + """ + self.render_config_template() + exit_code = self.run_beat( + logging_args=[], + extra_args=["export", "index-pattern", "-E", "migration.enabled:true"]) + + assert exit_code == 0 + assert self.log_contains('"objects": [') + assert self.log_contains('beat.name') diff --git a/libbeat/cmd/export.go b/libbeat/cmd/export.go index 2bd37f33adb..c1d059bcfe0 100644 --- a/libbeat/cmd/export.go +++ b/libbeat/cmd/export.go @@ -32,6 +32,7 @@ func genExportCmd(settings instance.Settings, name, idxPrefix, beatVersion strin exportCmd.AddCommand(export.GenExportConfigCmd(settings, name, idxPrefix, beatVersion)) exportCmd.AddCommand(export.GenTemplateConfigCmd(settings, name, idxPrefix, beatVersion)) + exportCmd.AddCommand(export.GenIndexPatternConfigCmd(settings, name, idxPrefix, beatVersion)) exportCmd.AddCommand(export.GenDashboardCmd(name, idxPrefix, beatVersion)) exportCmd.AddCommand(export.GenGetILMPolicyCmd(settings, name, idxPrefix, beatVersion)) diff --git a/libbeat/cmd/export/index_pattern.go b/libbeat/cmd/export/index_pattern.go new file mode 100644 index 00000000000..cf9a3a4f684 --- /dev/null +++ b/libbeat/cmd/export/index_pattern.go @@ -0,0 +1,87 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package export + +import ( + "log" + "os" + + "github.com/spf13/cobra" + + "github.com/elastic/beats/libbeat/cmd/instance" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/kibana" +) + +// GenIndexPatternConfigCmd generates an index pattern for Kibana +func GenIndexPatternConfigCmd(settings instance.Settings, name, idxPrefix, beatVersion string) *cobra.Command { + genTemplateConfigCmd := &cobra.Command{ + Use: "index-pattern", + Short: "Export kibana index pattern to stdout", + Run: func(cmd *cobra.Command, args []string) { + version, _ := cmd.Flags().GetString("es.version") + + b, err := instance.NewBeat(name, idxPrefix, beatVersion) + if err != nil { + fatalf("Error initializing beat: %+v", err) + } + err = b.InitWithSettings(settings) + if err != nil { + fatalf("Error initializing beat: %+v", err) + } + + if version == "" { + version = b.Info.Version + } + + var withMigration bool + if b.RawConfig.HasField("migration") { + sub, err := b.RawConfig.Child("migration", -1) + if err != nil { + fatalf("Failed to read migration setting: %+v", err) + } + withMigration = sub.Enabled() + } + + // Index pattern generation + v, err := common.NewVersion(version) + if err != nil { + fatalf("Error creating version: %+v", err) + } + indexPattern, err := kibana.NewGenerator(b.Info.IndexPrefix, b.Info.Beat, b.Fields, beatVersion, *v, withMigration) + if err != nil { + log.Fatal(err) + } + + pattern, err := indexPattern.Generate() + if err != nil { + log.Fatalf("ERROR: %s", err) + } + + _, err = os.Stdout.WriteString(pattern.StringToPrint() + "\n") + if err != nil { + fatalf("Error writing index pattern: %+v", err) + } + }, + } + + genTemplateConfigCmd.Flags().String("es.version", beatVersion, "Elasticsearch version") + genTemplateConfigCmd.Flags().String("index", idxPrefix, "Base index name") + + return genTemplateConfigCmd +} diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 765bc5316a4..6a64e10d3b0 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -32,6 +32,8 @@ import ( "strings" "time" + "github.com/elastic/beats/libbeat/kibana" + "github.com/gofrs/uuid" errw "github.com/pkg/errors" "go.uber.org/zap" @@ -679,8 +681,42 @@ func (b *Beat) loadDashboards(ctx context.Context, force bool) error { } if b.Config.Dashboards.Enabled() { - err := dashboards.ImportDashboards(ctx, b.Info.Beat, b.Info.Hostname, paths.Resolve(paths.Home, ""), - b.Config.Kibana, b.Config.Dashboards, nil) + + var withMigration bool + if b.RawConfig.HasField("migration") { + sub, err := b.RawConfig.Child("migration", -1) + if err != nil { + return fmt.Errorf("Failed to read migration setting: %+v", err) + } + withMigration = sub.Enabled() + } + + // init kibana config object + kibanaConfig := b.Config.Kibana + if kibanaConfig == nil { + kibanaConfig = common.NewConfig() + } + + client, err := kibana.NewKibanaClient(kibanaConfig) + if err != nil { + return fmt.Errorf("error connecting to Kibana: %v", err) + } + // This fetches the version for Kibana. For the alias feature the version of ES would be needed + // but it's assumed that KB and ES have the same minor version. + v := client.GetVersion() + + indexPattern, err := kibana.NewGenerator(b.Info.IndexPrefix, b.Info.Beat, b.Fields, b.Info.Version, v, withMigration) + if err != nil { + return fmt.Errorf("error creating index pattern generator: %v", err) + } + + pattern, err := indexPattern.Generate() + if err != nil { + return fmt.Errorf("error generating index pattern: %v", err) + } + + err = dashboards.ImportDashboards(ctx, b.Info, paths.Resolve(paths.Home, ""), + kibanaConfig, b.Config.Dashboards, nil, pattern) if err != nil { return errw.Wrap(err, "Error importing Kibana dashboards") } diff --git a/libbeat/common/field.go b/libbeat/common/field.go index 2decdd41092..1686d1c3a78 100644 --- a/libbeat/common/field.go +++ b/libbeat/common/field.go @@ -119,7 +119,7 @@ func (f *Field) Validate() error { } func LoadFieldsYaml(path string) (Fields, error) { - keys := []Field{} + var keys []Field cfg, err := yaml.NewConfigWithFile(path) if err != nil { @@ -135,6 +135,23 @@ func LoadFieldsYaml(path string) (Fields, error) { return fields, nil } +// LoadFields loads fields from a byte array +func LoadFields(f []byte) (Fields, error) { + var keys []Field + + cfg, err := yaml.NewConfig(f) + if err != nil { + return nil, err + } + cfg.Unpack(&keys) + + fields := Fields{} + for _, key := range keys { + fields = append(fields, key.Fields...) + } + return fields, nil +} + // HasKey checks if inside fields the given key exists // The key can be in the form of a.b.c and it will check if the nested field exist // In case the key is `a` and there is a value `a.b` false is return as it only diff --git a/libbeat/dashboards/dashboards.go b/libbeat/dashboards/dashboards.go index 79d664c3db8..c1ba0ca9b1c 100644 --- a/libbeat/dashboards/dashboards.go +++ b/libbeat/dashboards/dashboards.go @@ -25,15 +25,17 @@ import ( errw "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" ) // ImportDashboards tries to import the kibana dashboards. func ImportDashboards( ctx context.Context, - beatName, hostname, homePath string, + beatInfo beat.Info, homePath string, kibanaConfig, dashboardsConfig *common.Config, msgOutputter MessageOutputter, + pattern common.MapStr, ) error { if dashboardsConfig == nil || !dashboardsConfig.Enabled() { return nil @@ -41,28 +43,22 @@ func ImportDashboards( // unpack dashboard config dashConfig := defaultConfig - dashConfig.Beat = beatName + dashConfig.Beat = beatInfo.Beat dashConfig.Dir = filepath.Join(homePath, defaultDirectory) err := dashboardsConfig.Unpack(&dashConfig) if err != nil { return err } - // init kibana config object - if kibanaConfig == nil { - kibanaConfig = common.NewConfig() - } - if !kibanaConfig.Enabled() { return errors.New("kibana configuration missing for loading dashboards.") } - return setupAndImportDashboardsViaKibana(ctx, hostname, kibanaConfig, &dashConfig, msgOutputter) - + return setupAndImportDashboardsViaKibana(ctx, beatInfo.Hostname, kibanaConfig, &dashConfig, msgOutputter, pattern) } func setupAndImportDashboardsViaKibana(ctx context.Context, hostname string, kibanaConfig *common.Config, - dashboardsConfig *Config, msgOutputter MessageOutputter) error { + dashboardsConfig *Config, msgOutputter MessageOutputter, fields common.MapStr) error { kibanaLoader, err := NewKibanaLoader(ctx, kibanaConfig, dashboardsConfig, hostname, msgOutputter) if err != nil { @@ -73,10 +69,11 @@ func setupAndImportDashboardsViaKibana(ctx context.Context, hostname string, kib kibanaLoader.statusMsg("Kibana URL %v", kibanaLoader.client.Connection.URL) - return ImportDashboardsViaKibana(kibanaLoader) + return ImportDashboardsViaKibana(kibanaLoader, fields) } -func ImportDashboardsViaKibana(kibanaLoader *KibanaLoader) error { +// ImportDashboardsViaKibana imports Dashboards to Kibana +func ImportDashboardsViaKibana(kibanaLoader *KibanaLoader, fields common.MapStr) error { version := kibanaLoader.version if !version.IsValid() { return errors.New("No valid kibana version available") @@ -86,7 +83,7 @@ func ImportDashboardsViaKibana(kibanaLoader *KibanaLoader) error { return fmt.Errorf("Kibana API is not available in Kibana version %s", kibanaLoader.version.String()) } - importer, err := NewImporter(version, kibanaLoader.config, kibanaLoader) + importer, err := NewImporter(version, kibanaLoader.config, *kibanaLoader, fields) if err != nil { return fmt.Errorf("fail to create a Kibana importer for loading the dashboards: %v", err) } diff --git a/libbeat/dashboards/importer.go b/libbeat/dashboards/importer.go index 8a1b3bc6c93..a9427651eb2 100644 --- a/libbeat/dashboards/importer.go +++ b/libbeat/dashboards/importer.go @@ -54,17 +54,12 @@ type Importer struct { cfg *Config version common.Version - loader Loader + loader KibanaLoader + fields common.MapStr } -type Loader interface { - ImportIndex(file string) error - ImportDashboard(file string) error - statusMsg(msg string, a ...interface{}) - Close() error -} - -func NewImporter(version common.Version, cfg *Config, loader Loader) (*Importer, error) { +// NewImporter creates a new dashboard importer +func NewImporter(version common.Version, cfg *Config, loader KibanaLoader, fields common.MapStr) (*Importer, error) { // Current max version is 7 if version.Major > 6 { @@ -75,6 +70,7 @@ func NewImporter(version common.Version, cfg *Config, loader Loader) (*Importer, cfg: cfg, version: version, loader: loader, + fields: fields, }, nil } @@ -106,7 +102,7 @@ func (imp Importer) ImportFile(fileType string, file string) error { if fileType == "dashboard" { return imp.loader.ImportDashboard(file) } else if fileType == "index-pattern" { - return imp.loader.ImportIndex(file) + return imp.loader.ImportIndexFile(file) } return fmt.Errorf("Unexpected file type %s", fileType) } @@ -310,6 +306,10 @@ func (imp Importer) ImportKibanaDir(dir string) error { return newErrNotFound("No directory %s", dir) } + // Loads the internal index pattern + if imp.fields != nil { + imp.loader.ImportIndex(imp.fields) + } check := []string{} if !imp.cfg.OnlyDashboards { check = append(check, "index-pattern") diff --git a/libbeat/dashboards/kibana_loader.go b/libbeat/dashboards/kibana_loader.go index 6c6e291e4ae..ca7dd345f15 100644 --- a/libbeat/dashboards/kibana_loader.go +++ b/libbeat/dashboards/kibana_loader.go @@ -40,6 +40,7 @@ type KibanaLoader struct { msgOutputter MessageOutputter } +// NewKibanaLoader creates a new loader to load Kibana files func NewKibanaLoader(ctx context.Context, cfg *common.Config, dashboardsConfig *Config, hostname string, msgOutputter MessageOutputter) (*KibanaLoader, error) { if cfg == nil || !cfg.Enabled() { @@ -81,10 +82,8 @@ func getKibanaClient(ctx context.Context, cfg *common.Config, retryCfg *Retry, r return client, nil } -func (loader KibanaLoader) ImportIndex(file string) error { - params := url.Values{} - params.Set("force", "true") //overwrite the existing dashboards - +// ImportIndexFile imports an index pattern from a file +func (loader KibanaLoader) ImportIndexFile(file string) error { // read json file reader, err := ioutil.ReadFile(file) if err != nil { @@ -97,11 +96,19 @@ func (loader KibanaLoader) ImportIndex(file string) error { return fmt.Errorf("fail to unmarshal the index content from file %s: %v", file, err) } - indexContent = ReplaceIndexInIndexPattern(loader.config.Index, indexContent) + return loader.ImportIndex(indexContent) +} + +// ImportIndex imports the passed index pattern to Kibana +func (loader KibanaLoader) ImportIndex(pattern common.MapStr) error { + params := url.Values{} + params.Set("force", "true") //overwrite the existing dashboards + indexContent := ReplaceIndexInIndexPattern(loader.config.Index, pattern) return loader.client.ImportJSON(importAPI, params, indexContent) } +// ImportDashboard imports the dashboard file func (loader KibanaLoader) ImportDashboard(file string) error { params := url.Values{} params.Set("force", "true") //overwrite the existing dashboards diff --git a/libbeat/kibana/fields_transformer.go b/libbeat/kibana/fields_transformer.go index 3ccf212c362..207c193a750 100644 --- a/libbeat/kibana/fields_transformer.go +++ b/libbeat/kibana/fields_transformer.go @@ -32,9 +32,10 @@ type fieldsTransformer struct { transformedFieldFormatMap common.MapStr version *common.Version keys map[string]int + migration bool } -func newFieldsTransformer(version *common.Version, fields common.Fields) (*fieldsTransformer, error) { +func newFieldsTransformer(version *common.Version, fields common.Fields, migration bool) (*fieldsTransformer, error) { if version == nil { return nil, errors.New("Version must be given") } @@ -44,6 +45,7 @@ func newFieldsTransformer(version *common.Version, fields common.Fields) (*field transformedFields: []common.MapStr{}, transformedFieldFormatMap: common.MapStr{}, keys: map[string]int{}, + migration: migration, }, nil } @@ -90,6 +92,10 @@ func (t *fieldsTransformer) transformFields(commonFields common.Fields, path str if t.version.LessThan(v640) { continue } + // Only adds migration aliases if migration is enabled + if f.MigrationAlias && !t.migration { + continue + } if ff := t.fields.GetField(f.AliasPath); ff != nil { // copy the field, keep path := f.Path diff --git a/libbeat/kibana/fields_transformer_test.go b/libbeat/kibana/fields_transformer_test.go index 90122667448..1fb1e8b3a0b 100644 --- a/libbeat/kibana/fields_transformer_test.go +++ b/libbeat/kibana/fields_transformer_test.go @@ -35,7 +35,7 @@ var ( ) func TestEmpty(t *testing.T) { - trans, err := newFieldsTransformer(version, common.Fields{}) + trans, err := newFieldsTransformer(version, common.Fields{}, true) assert.NoError(t, err) out, err := trans.transform() assert.NoError(t, err) @@ -93,7 +93,7 @@ func TestEmpty(t *testing.T) { func TestMissingVersion(t *testing.T) { var c *common.Version - _, err := newFieldsTransformer(c, common.Fields{}) + _, err := newFieldsTransformer(c, common.Fields{}, true) assert.Error(t, err) } @@ -118,9 +118,10 @@ func TestDuplicateField(t *testing.T) { }}, } for _, testCase := range testCases { - trans, err := newFieldsTransformer(version, testCase.commonFields) + trans, err := newFieldsTransformer(version, testCase.commonFields, true) require.NoError(t, err) _, err = trans.transform() + fmt.Println(err) assert.Error(t, err) } } @@ -151,7 +152,7 @@ func TestValidDuplicateField(t *testing.T) { }, }, } - trans, err := newFieldsTransformer(version, commonFields) + trans, err := newFieldsTransformer(version, commonFields, true) require.NoError(t, err) transformed, err := trans.transform() require.NoError(t, err) @@ -179,7 +180,7 @@ func TestInvalidVersion(t *testing.T) { }, }, } - trans, err := newFieldsTransformer(version, commonFields) + trans, err := newFieldsTransformer(version, commonFields, true) assert.NoError(t, err) _, err = trans.transform() assert.Error(t, err) @@ -206,7 +207,7 @@ func TestTransformTypes(t *testing.T) { {commonField: common.Field{Type: "invalid"}, expected: nil}, } for idx, test := range tests { - trans, _ := newFieldsTransformer(version, common.Fields{test.commonField}) + trans, _ := newFieldsTransformer(version, common.Fields{test.commonField}, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fields"].([]common.MapStr)[0] @@ -251,7 +252,7 @@ func TestTransformGroup(t *testing.T) { }, } for idx, test := range tests { - trans, _ := newFieldsTransformer(version, test.commonFields) + trans, _ := newFieldsTransformer(version, test.commonFields, false) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fields"].([]common.MapStr) @@ -327,7 +328,7 @@ func TestTransformMisc(t *testing.T) { {commonField: common.Field{Script: "doc[]"}, expected: "painless", attr: "lang"}, } for idx, test := range tests { - trans, _ := newFieldsTransformer(version, common.Fields{test.commonField}) + trans, _ := newFieldsTransformer(version, common.Fields{test.commonField}, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fields"].([]common.MapStr)[0] @@ -526,7 +527,7 @@ func TestTransformFieldFormatMap(t *testing.T) { }, } for idx, test := range tests { - trans, _ := newFieldsTransformer(test.version, common.Fields{test.commonField}) + trans, _ := newFieldsTransformer(test.version, common.Fields{test.commonField}, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fieldFormatMap"] @@ -594,7 +595,7 @@ func TestTransformGroupAndEnabled(t *testing.T) { }, } for idx, test := range tests { - trans, _ := newFieldsTransformer(version, test.commonFields) + trans, _ := newFieldsTransformer(version, test.commonFields, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fields"].([]common.MapStr) @@ -614,7 +615,7 @@ func TestTransformMultiField(t *testing.T) { common.Field{Name: "text", Type: "text"}, }, } - trans, _ := newFieldsTransformer(version, common.Fields{f}) + trans, _ := newFieldsTransformer(version, common.Fields{f}, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fields"].([]common.MapStr) diff --git a/libbeat/kibana/index_pattern_generator.go b/libbeat/kibana/index_pattern_generator.go index d96646e7ad6..8252177d27f 100644 --- a/libbeat/kibana/index_pattern_generator.go +++ b/libbeat/kibana/index_pattern_generator.go @@ -28,45 +28,34 @@ import ( ) type IndexPatternGenerator struct { - indexName string - beatVersion string - fieldsYaml string - version common.Version - targetDir string - targetFilename string + indexName string + beatVersion string + fields []byte + version common.Version + migration bool } // Create an instance of the Kibana Index Pattern Generator -func NewGenerator(indexName, beatName, fieldsYAMLFile, outputDir, beatVersion string, version common.Version) (*IndexPatternGenerator, error) { +func NewGenerator(indexName, beatName string, fields []byte, beatVersion string, version common.Version, migration bool) (*IndexPatternGenerator, error) { beatName = clean(beatName) - if _, err := os.Stat(fieldsYAMLFile); err != nil { - return nil, err - } - return &IndexPatternGenerator{ - indexName: indexName, - fieldsYaml: fieldsYAMLFile, - beatVersion: beatVersion, - version: version, - targetDir: createTargetDir(outputDir, version), - targetFilename: beatName + ".json", + indexName: indexName + "-*", + fields: fields, + beatVersion: beatVersion, + version: version, + migration: migration, }, nil } // Generate creates the Index-Pattern for Kibana. -func (i *IndexPatternGenerator) Generate() (string, error) { +func (i *IndexPatternGenerator) Generate() (common.MapStr, error) { idxPattern, err := i.generate() if err != nil { - return "", err + return nil, err } - idxPattern = i.generatePattern(idxPattern) - - file := filepath.Join(i.targetDir, i.targetFilename) - err = dumpToFile(file, idxPattern) - - return file, err + return i.generatePattern(idxPattern), nil } func (i *IndexPatternGenerator) generate() (common.MapStr, error) { @@ -105,7 +94,7 @@ func (i *IndexPatternGenerator) generatePattern(attrs common.MapStr) common.MapS } func (i *IndexPatternGenerator) addGeneral(indexPattern *common.MapStr) error { - kibanaEntries, err := loadKibanaEntriesFromYaml(i.fieldsYaml) + kibanaEntries, err := loadKibanaEntriesFromYaml(i.fields) if err != nil { return err } @@ -121,11 +110,11 @@ func (i *IndexPatternGenerator) addGeneral(indexPattern *common.MapStr) error { } func (i *IndexPatternGenerator) addFieldsSpecific(indexPattern *common.MapStr) error { - fields, err := common.LoadFieldsYaml(i.fieldsYaml) + fields, err := common.LoadFields(i.fields) if err != nil { return err } - transformer, err := newFieldsTransformer(&i.version, fields) + transformer, err := newFieldsTransformer(&i.version, fields, i.migration) if err != nil { return err } diff --git a/libbeat/kibana/index_pattern_generator_test.go b/libbeat/kibana/index_pattern_generator_test.go index 302eaafb693..a703f0e419a 100644 --- a/libbeat/kibana/index_pattern_generator_test.go +++ b/libbeat/kibana/index_pattern_generator_test.go @@ -22,7 +22,6 @@ import ( "fmt" "io/ioutil" "os" - "path/filepath" "strings" "testing" @@ -39,27 +38,20 @@ func TestNewGenerator(t *testing.T) { tmpDir := tmpPath(t) defer os.RemoveAll(tmpDir) - v, _ := common.NewVersion("7.0.0") - // checks for fields.yml - generator, err := NewGenerator("beat-index", "mybeat.", fieldsYml+".missing", tmpDir, "7.0", *v) - assert.Error(t, err) - - generator, err = NewGenerator("beat-index", "mybeat.", fieldsYml, tmpDir, "7.0", *v) + data, err := ioutil.ReadFile("./testdata/fields.yml") if err != nil { t.Fatal(err) } - assert.Equal(t, "7.0", generator.beatVersion) - assert.Equal(t, "beat-index", generator.indexName) - // creates file dir and sets name - expectedDir := filepath.Join(tmpDir, "7/index-pattern") - assert.Equal(t, expectedDir, generator.targetDir) - _, err = os.Stat(generator.targetDir) + v, _ := common.NewVersion("7.0.0") + // checks for fields.yml + generator, err := NewGenerator("beat-index", "mybeat.", data, "7.0", *v, true) if err != nil { t.Fatal(err) } + assert.Equal(t, "7.0", generator.beatVersion) + assert.Equal(t, "beat-index-*", generator.indexName) - assert.Equal(t, "mybeat.json", generator.targetFilename) } func TestCleanName(t *testing.T) { @@ -78,129 +70,102 @@ func TestCleanName(t *testing.T) { } } -func TestGenerateFieldsYaml(t *testing.T) { - tmpDir := tmpPath(t) - defer os.RemoveAll(tmpDir) - - v, _ := common.NewVersion("6.0.0") - generator, err := NewGenerator("metricbeat-*", "metric beat ?!", fieldsYml, tmpDir, "7.0.0-alpha1", *v) - if err != nil { - t.Fatal(err) - } - - _, err = generator.Generate() - if err != nil { - t.Fatal(err) - } - - generator.fieldsYaml = "" - _, err = generator.Generate() - assert.Error(t, err) -} - -func TestDumpToFileDefault(t *testing.T) { - tmpDir := tmpPath(t) - defer os.RemoveAll(tmpDir) - - v, _ := common.NewVersion("7.0.0") - generator, err := NewGenerator("metricbeat-*", "metric beat ?!", fieldsYml, tmpDir, "7.0.0-alpha1", *v) - if err != nil { - t.Fatal(err) - } - - _, err = generator.Generate() - if err != nil { - t.Fatal(err) - } - - generator.targetDir = filepath.Join(tmpDir, "./non-existing/something") - _, err = generator.Generate() - assert.Error(t, err) -} - func TestGenerate(t *testing.T) { tmpDir := tmpPath(t) defer os.RemoveAll(tmpDir) - v6, _ := common.NewVersion("6.4.0") + v6, _ := common.NewVersion("7.0.0-alpha1") versions := []*common.Version{v6} + var d common.MapStr for _, version := range versions { - generator, err := NewGenerator("beat-*", "b eat ?!", fieldsYml, tmpDir, "7.0.0-alpha1", *version) + data, err := ioutil.ReadFile("./testdata/fields.yml") + if err != nil { + t.Fatal(err) + } + generator, err := NewGenerator("beat", "b eat ?!", data, version.String(), *version, true) if err != nil { t.Fatal(err) } - _, err = generator.Generate() + d, err = generator.Generate() if err != nil { t.Fatal(err) } } - tests := []map[string]string{ + tests := []ttt{ { - "existing": "testdata/beat-6.json", - "created": filepath.Join(tmpDir, "7/index-pattern/beat.json"), + existing: "testdata/beat-6.json", + created: d, }, } + testGenerate(t, tests, true) } +type ttt struct { + existing string + created common.MapStr +} + func TestGenerateExtensive(t *testing.T) { tmpDir := tmpPath(t) defer os.RemoveAll(tmpDir) - version6, _ := common.NewVersion("6.4.0") + version6, _ := common.NewVersion("7.0.0-alpha1") versions := []*common.Version{version6} + + var d common.MapStr for _, version := range versions { - generator, err := NewGenerator("metricbeat-*", "metric be at ?!", "testdata/extensive/fields.yml", tmpDir, "7.0.0-alpha1", *version) + data, err := ioutil.ReadFile("testdata/extensive/fields.yml") + if err != nil { + t.Fatal(err) + } + generator, err := NewGenerator("metricbeat", "metric be at ?!", data, version.String(), *version, true) if err != nil { t.Fatal(err) } - _, err = generator.Generate() + d, err = generator.Generate() if err != nil { t.Fatal(err) } } - tests := []map[string]string{ + tests := []ttt{ { - "existing": "testdata/extensive/metricbeat-6.json", - "created": filepath.Join(tmpDir, "7/index-pattern/metricbeat.json"), + existing: "testdata/extensive/metricbeat-6.json", + created: d, }, } testGenerate(t, tests, false) } -func testGenerate(t *testing.T, tests []map[string]string, sourceFilters bool) { +func testGenerate(t *testing.T, tests []ttt, sourceFilters bool) { for _, test := range tests { // compare default - existing, err := readJson(test["existing"]) - if err != nil { - t.Fatal(err) - } - created, err := readJson(test["created"]) + existing, err := readJson(test.existing) if err != nil { t.Fatal(err) } var attrExisting, attrCreated common.MapStr - if strings.Contains(test["existing"], "6") { - assert.Equal(t, existing["version"], created["version"]) + if strings.Contains(test.existing, "6") { + assert.Equal(t, existing["version"], test.created["version"]) objExisting := existing["objects"].([]interface{})[0].(map[string]interface{}) - objCreated := created["objects"].([]interface{})[0].(map[string]interface{}) + objCreated := test.created["objects"].([]common.MapStr)[0] - assert.Equal(t, objExisting["version"], objCreated["version"]) + assert.Equal(t, int(objExisting["version"].(float64)), objCreated["version"]) assert.Equal(t, objExisting["id"], objCreated["id"]) assert.Equal(t, objExisting["type"], objCreated["type"]) attrExisting = objExisting["attributes"].(map[string]interface{}) - attrCreated = objCreated["attributes"].(map[string]interface{}) + attrCreated = objCreated["attributes"].(common.MapStr) } else { attrExisting = existing - attrCreated = created + attrCreated = test.created } // check fieldFormatMap diff --git a/libbeat/kibana/transformer.go b/libbeat/kibana/transformer.go index 9c4649e2cec..f99dd5ff0c0 100644 --- a/libbeat/kibana/transformer.go +++ b/libbeat/kibana/transformer.go @@ -50,9 +50,9 @@ func (t *transformer) transform() common.MapStr { return transformed } -func loadKibanaEntriesFromYaml(yamlFile string) ([]kibanaEntry, error) { +func loadKibanaEntriesFromYaml(fields []byte) ([]kibanaEntry, error) { entries := []kibanaEntry{} - cfg, err := yaml.NewConfigWithFile(yamlFile) + cfg, err := yaml.NewConfig(fields) if err != nil { return nil, err } diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 47196dd5f9c..6beccd87139 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -360,9 +360,6 @@ endif fi @# Convert all dashboards to string @python ${ES_BEATS}/libbeat/scripts/unpack_dashboards.py --glob="./_meta/kibana.generated/7/dashboard/*.json" - @go run ${ES_BEATS}/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go \ - -index '${BEAT_INDEX_PREFIX}-*' -beat ${BEAT_NAME} -fields $(PWD)/fields.yml \ - -version ${BEAT_VERSION} -out _meta/kibana.generated endif .PHONY: docs diff --git a/libbeat/tests/system/test_index_pattern.py b/libbeat/tests/system/test_index_pattern.py new file mode 100644 index 00000000000..85f86684e7b --- /dev/null +++ b/libbeat/tests/system/test_index_pattern.py @@ -0,0 +1,21 @@ +from base import BaseTest +import os + + +class Test(BaseTest): + + def test_export_index_pattern(self): + """ + Test export index pattern + """ + self.render_config_template("mockbeat", + os.path.join(self.working_dir, + "mockbeat.yml"), + fields=os.path.join(self.working_dir, "fields.yml")) + exit_code = self.run_beat( + logging_args=[], + extra_args=["export", "index-pattern"], + config="mockbeat.yml") + + assert exit_code == 0 + assert self.log_contains('"objects": [') diff --git a/metricbeat/Makefile b/metricbeat/Makefile index a034e954b44..25316f80bff 100644 --- a/metricbeat/Makefile +++ b/metricbeat/Makefile @@ -78,5 +78,5 @@ test-module: python-env update metricbeat.test assets: go run ${ES_BEATS}/metricbeat/scripts/assets/assets.go ${ES_BEATS}/metricbeat/module mkdir -p include/fields - go run ${ES_BEATS}/dev-tools/cmd/asset/asset.go -license ${LICENSE} -pkg include -in ${ES_BEATS}/metricbeat/_meta/fields.common.yml -out include/fields.go $(BEAT_NAME) + #go run ${ES_BEATS}/dev-tools/cmd/asset/asset.go -license ${LICENSE} -pkg include -in ${ES_BEATS}/metricbeat/_meta/fields.common.yml -out include/fields.go $(BEAT_NAME) go run ${ES_BEATS}/libbeat/scripts/cmd/global_fields/main.go -es_beats_path ${ES_BEATS} -beat_path ${PWD} | go run ${ES_BEATS}/dev-tools/cmd/asset/asset.go -license ${LICENSE} -out ./include/fields/fields.go -pkg include -priority asset.LibbeatFieldsPri ${ES_BEATS}/libbeat/fields.yml $(BEAT_NAME) diff --git a/metricbeat/include/fields.go b/metricbeat/include/fields.go deleted file mode 100644 index a1115095c3c..00000000000 --- a/metricbeat/include/fields.go +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. - -package include - -import ( - "github.com/elastic/beats/libbeat/asset" -) - -func init() { - if err := asset.SetFields("metricbeat", "../metricbeat/_meta/fields.common.yml", asset.BeatFieldsPri, AssetMetricbeatMetaFieldsCommonYml); err != nil { - panic(err) - } -} - -// AssetMetricbeatMetaFieldsCommonYml returns asset data. -// This is the base64 encoded gzipped contents of ../metricbeat/_meta/fields.common.yml. -func AssetMetricbeatMetaFieldsCommonYml() string { - return "eJyUU0tu3DAM3fsUD9nHB/CiQJBNl13kAozEWET1cUV6pr59YVkT1EUzab0TxfcR+fyI77xNcCWlkgfAxCJPeL6dPaurspiUPOHLAADPJRtJ1g7Cm3D0CrqQRHqNDMmgGMEXzgbbFtZxQG+bhsbxiEyJJyS2Kk7ZxlT8Grld/lV1/14CNxzKGywwDgwskGHmzJWMfbtp2mMH7hYmUBTSXlnIwtSbTsJJ5kqHrNWVPzK7n//T6g17x+1JbKnFseq4zOJP74glz/e1vx1QzLWsC8T/waxcL+J4JO8rq97nejqa3p9BLkhmXANXbpXOBlHUNWfJ84iXIPrO0PaORBtyMbwylsq6J+MaODcKT0a40h6oGNntU4nFUYzbB85DUft8BV+L2nkH/2L+rLiPvBNW/rFKZd+TcRT5J6Ul/p6NT+a558IXt6bbvzHiKV5pU7RsFDz44h7G4VcAAAD//8J8HrM=" -} From c3341dbad2afca9673c400ae56a56b097c262ed5 Mon Sep 17 00:00:00 2001 From: beats-jenkins Date: Mon, 4 Feb 2019 09:25:59 +0100 Subject: [PATCH 2/2] cleanup tests and makefile --- libbeat/kibana/index_pattern_generator_test.go | 16 ++++++++-------- metricbeat/Makefile | 1 - 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/libbeat/kibana/index_pattern_generator_test.go b/libbeat/kibana/index_pattern_generator_test.go index a703f0e419a..38cdc2dc983 100644 --- a/libbeat/kibana/index_pattern_generator_test.go +++ b/libbeat/kibana/index_pattern_generator_test.go @@ -74,8 +74,8 @@ func TestGenerate(t *testing.T) { tmpDir := tmpPath(t) defer os.RemoveAll(tmpDir) - v6, _ := common.NewVersion("7.0.0-alpha1") - versions := []*common.Version{v6} + v7, _ := common.NewVersion("7.0.0-alpha1") + versions := []*common.Version{v7} var d common.MapStr for _, version := range versions { data, err := ioutil.ReadFile("./testdata/fields.yml") @@ -93,7 +93,7 @@ func TestGenerate(t *testing.T) { } } - tests := []ttt{ + tests := []compare{ { existing: "testdata/beat-6.json", created: d, @@ -103,7 +103,7 @@ func TestGenerate(t *testing.T) { testGenerate(t, tests, true) } -type ttt struct { +type compare struct { existing string created common.MapStr } @@ -112,8 +112,8 @@ func TestGenerateExtensive(t *testing.T) { tmpDir := tmpPath(t) defer os.RemoveAll(tmpDir) - version6, _ := common.NewVersion("7.0.0-alpha1") - versions := []*common.Version{version6} + version7, _ := common.NewVersion("7.0.0-alpha1") + versions := []*common.Version{version7} var d common.MapStr for _, version := range versions { @@ -132,7 +132,7 @@ func TestGenerateExtensive(t *testing.T) { } } - tests := []ttt{ + tests := []compare{ { existing: "testdata/extensive/metricbeat-6.json", created: d, @@ -141,7 +141,7 @@ func TestGenerateExtensive(t *testing.T) { testGenerate(t, tests, false) } -func testGenerate(t *testing.T, tests []ttt, sourceFilters bool) { +func testGenerate(t *testing.T, tests []compare, sourceFilters bool) { for _, test := range tests { // compare default existing, err := readJson(test.existing) diff --git a/metricbeat/Makefile b/metricbeat/Makefile index 25316f80bff..710abbe9f89 100644 --- a/metricbeat/Makefile +++ b/metricbeat/Makefile @@ -78,5 +78,4 @@ test-module: python-env update metricbeat.test assets: go run ${ES_BEATS}/metricbeat/scripts/assets/assets.go ${ES_BEATS}/metricbeat/module mkdir -p include/fields - #go run ${ES_BEATS}/dev-tools/cmd/asset/asset.go -license ${LICENSE} -pkg include -in ${ES_BEATS}/metricbeat/_meta/fields.common.yml -out include/fields.go $(BEAT_NAME) go run ${ES_BEATS}/libbeat/scripts/cmd/global_fields/main.go -es_beats_path ${ES_BEATS} -beat_path ${PWD} | go run ${ES_BEATS}/dev-tools/cmd/asset/asset.go -license ${LICENSE} -out ./include/fields/fields.go -pkg include -priority asset.LibbeatFieldsPri ${ES_BEATS}/libbeat/fields.yml $(BEAT_NAME)