diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index eca8bfdc6a1..530b638a704 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -21,6 +21,7 @@ The list below covers the major changes between 7.0.0-alpha2 and master only. ==== Breaking changes - Outputs receive Index Manager as additional parameter. The index manager can be used to create an index selector. {pull}10347[10347] +- Remove support for loading dashboards to Elasticsearch 5. {pull}10451[10451] ==== Bugfixes @@ -31,3 +32,4 @@ The list below covers the major changes between 7.0.0-alpha2 and master only. - Add (*common.Config).Has and (*common.Config).Remove. {pull}10363[10363] - Introduce ILM and IndexManagment support to beat.Settings. {pull}10347[10347] - Introduce ILM and IndexManagement support to beat.Settings. {pull}10347[10347] +- Generating index pattern on demand instead of shipping them in the packages. {pull}10478[10478] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 46a95df13b4..daeef17620a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -203,6 +203,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add ILM mode `auto` to setup.ilm.enabled setting. This new default value detects if ILM is available {pull}10347[10347] - Add support to read ILM policy from external JSON file. {pull}10347[10347] - Add `overwrite` and `check_exists` settings to ILM support. {pull}10347[10347] +- Generate Kibana index pattern on demand instead of using a local file. {pull}10478[10478] *Auditbeat* diff --git a/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go b/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go deleted file mode 100644 index 4be7a8d808c..00000000000 --- a/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go +++ /dev/null @@ -1,92 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package main - -import ( - "flag" - "log" - "path/filepath" - - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/kibana" - "github.com/elastic/beats/libbeat/version" -) - -var usageText = ` -Usage: kibana_index_pattern [flags] - kibana_index_pattern generates Kibana index patterns from the Beat's - fields.yml file. It will create a index pattern file that is usable with both - Kibana 6.x and 7.x. -Options: -`[1:] - -var ( - beatName string - beatVersion string - indexPattern string - fieldsYAMLFile string - outputDir string -) - -func init() { - flag.StringVar(&beatName, "beat", "", "Name of the beat. (Required)") - flag.StringVar(&beatVersion, "version", version.GetDefaultVersion(), "Beat version. (Required)") - flag.StringVar(&indexPattern, "index", "", "Kibana index pattern. (Required)") - flag.StringVar(&fieldsYAMLFile, "fields", "fields.yml", "fields.yml file containing all fields used by the Beat.") - flag.StringVar(&outputDir, "out", "build/kibana", "Output dir.") -} - -func main() { - log.SetFlags(0) - flag.Parse() - - if beatName == "" { - log.Fatal("Name of the Beat must be set (-beat).") - } - - if beatVersion == "" { - log.Fatal("Beat version must be set (-version).") - } - - if indexPattern == "" { - log.Fatal("Index pattern must be set (-index).") - } - - versions := []string{ - "6.0.0", - } - for _, version := range versions { - version, _ := common.NewVersion(version) - indexPattern, err := kibana.NewGenerator(indexPattern, beatName, fieldsYAMLFile, outputDir, beatVersion, *version) - if err != nil { - log.Fatal(err) - } - - file, err := indexPattern.Generate() - if err != nil { - log.Fatalf("ERROR: %s", err) - } - - // Log output file location. - absFile, err := filepath.Abs(file) - if err != nil { - absFile = file - } - log.Printf(">> The index pattern was created under %v", absFile) - } -} diff --git a/dev-tools/mage/kibana.go b/dev-tools/mage/kibana.go index 1b1b7ecc80e..1240d1ca562 100644 --- a/dev-tools/mage/kibana.go +++ b/dev-tools/mage/kibana.go @@ -74,22 +74,7 @@ func KibanaDashboards(moduleDirs ...string) error { return err } - beatVersion, err := BeatQualifiedVersion() - if err != nil { - return err - } - - // Generate Kibana index pattern files from fields.yml. - indexPatternCmd := sh.RunCmd("go", "run", - filepath.Join(esBeatsDir, "dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go"), - "-beat", BeatName, - "-version", beatVersion, - "-index", BeatIndexPrefix+"-*", - "-fields", "fields.yml", - "-out", kibanaBuildDir, - ) - - return indexPatternCmd() + return nil } // PackageKibanaDashboardsFromBuildDir reconfigures the packaging configuration diff --git a/filebeat/tests/system/test_index_pattern.py b/filebeat/tests/system/test_index_pattern.py new file mode 100644 index 00000000000..69c0d43d20a --- /dev/null +++ b/filebeat/tests/system/test_index_pattern.py @@ -0,0 +1,32 @@ +import os +import unittest +from filebeat import BaseTest + + +class Test(BaseTest): + + def test_export_index_pattern(self): + """ + Test export index pattern + """ + self.render_config_template() + exit_code = self.run_beat( + logging_args=[], + extra_args=["export", "index-pattern"]) + + assert exit_code == 0 + assert self.log_contains('"objects": [') + assert self.log_contains('beat.name') == False + + def test_export_index_pattern_migration(self): + """ + Test export index pattern with migration flag enabled + """ + self.render_config_template() + exit_code = self.run_beat( + logging_args=[], + extra_args=["export", "index-pattern", "-E", "migration.enabled:true"]) + + assert exit_code == 0 + assert self.log_contains('"objects": [') + assert self.log_contains('beat.name') diff --git a/libbeat/cmd/export.go b/libbeat/cmd/export.go index 2bd37f33adb..c1d059bcfe0 100644 --- a/libbeat/cmd/export.go +++ b/libbeat/cmd/export.go @@ -32,6 +32,7 @@ func genExportCmd(settings instance.Settings, name, idxPrefix, beatVersion strin exportCmd.AddCommand(export.GenExportConfigCmd(settings, name, idxPrefix, beatVersion)) exportCmd.AddCommand(export.GenTemplateConfigCmd(settings, name, idxPrefix, beatVersion)) + exportCmd.AddCommand(export.GenIndexPatternConfigCmd(settings, name, idxPrefix, beatVersion)) exportCmd.AddCommand(export.GenDashboardCmd(name, idxPrefix, beatVersion)) exportCmd.AddCommand(export.GenGetILMPolicyCmd(settings, name, idxPrefix, beatVersion)) diff --git a/libbeat/cmd/export/index_pattern.go b/libbeat/cmd/export/index_pattern.go new file mode 100644 index 00000000000..cf9a3a4f684 --- /dev/null +++ b/libbeat/cmd/export/index_pattern.go @@ -0,0 +1,87 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package export + +import ( + "log" + "os" + + "github.com/spf13/cobra" + + "github.com/elastic/beats/libbeat/cmd/instance" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/kibana" +) + +// GenIndexPatternConfigCmd generates an index pattern for Kibana +func GenIndexPatternConfigCmd(settings instance.Settings, name, idxPrefix, beatVersion string) *cobra.Command { + genTemplateConfigCmd := &cobra.Command{ + Use: "index-pattern", + Short: "Export kibana index pattern to stdout", + Run: func(cmd *cobra.Command, args []string) { + version, _ := cmd.Flags().GetString("es.version") + + b, err := instance.NewBeat(name, idxPrefix, beatVersion) + if err != nil { + fatalf("Error initializing beat: %+v", err) + } + err = b.InitWithSettings(settings) + if err != nil { + fatalf("Error initializing beat: %+v", err) + } + + if version == "" { + version = b.Info.Version + } + + var withMigration bool + if b.RawConfig.HasField("migration") { + sub, err := b.RawConfig.Child("migration", -1) + if err != nil { + fatalf("Failed to read migration setting: %+v", err) + } + withMigration = sub.Enabled() + } + + // Index pattern generation + v, err := common.NewVersion(version) + if err != nil { + fatalf("Error creating version: %+v", err) + } + indexPattern, err := kibana.NewGenerator(b.Info.IndexPrefix, b.Info.Beat, b.Fields, beatVersion, *v, withMigration) + if err != nil { + log.Fatal(err) + } + + pattern, err := indexPattern.Generate() + if err != nil { + log.Fatalf("ERROR: %s", err) + } + + _, err = os.Stdout.WriteString(pattern.StringToPrint() + "\n") + if err != nil { + fatalf("Error writing index pattern: %+v", err) + } + }, + } + + genTemplateConfigCmd.Flags().String("es.version", beatVersion, "Elasticsearch version") + genTemplateConfigCmd.Flags().String("index", idxPrefix, "Base index name") + + return genTemplateConfigCmd +} diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 765bc5316a4..6a64e10d3b0 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -32,6 +32,8 @@ import ( "strings" "time" + "github.com/elastic/beats/libbeat/kibana" + "github.com/gofrs/uuid" errw "github.com/pkg/errors" "go.uber.org/zap" @@ -679,8 +681,42 @@ func (b *Beat) loadDashboards(ctx context.Context, force bool) error { } if b.Config.Dashboards.Enabled() { - err := dashboards.ImportDashboards(ctx, b.Info.Beat, b.Info.Hostname, paths.Resolve(paths.Home, ""), - b.Config.Kibana, b.Config.Dashboards, nil) + + var withMigration bool + if b.RawConfig.HasField("migration") { + sub, err := b.RawConfig.Child("migration", -1) + if err != nil { + return fmt.Errorf("Failed to read migration setting: %+v", err) + } + withMigration = sub.Enabled() + } + + // init kibana config object + kibanaConfig := b.Config.Kibana + if kibanaConfig == nil { + kibanaConfig = common.NewConfig() + } + + client, err := kibana.NewKibanaClient(kibanaConfig) + if err != nil { + return fmt.Errorf("error connecting to Kibana: %v", err) + } + // This fetches the version for Kibana. For the alias feature the version of ES would be needed + // but it's assumed that KB and ES have the same minor version. + v := client.GetVersion() + + indexPattern, err := kibana.NewGenerator(b.Info.IndexPrefix, b.Info.Beat, b.Fields, b.Info.Version, v, withMigration) + if err != nil { + return fmt.Errorf("error creating index pattern generator: %v", err) + } + + pattern, err := indexPattern.Generate() + if err != nil { + return fmt.Errorf("error generating index pattern: %v", err) + } + + err = dashboards.ImportDashboards(ctx, b.Info, paths.Resolve(paths.Home, ""), + kibanaConfig, b.Config.Dashboards, nil, pattern) if err != nil { return errw.Wrap(err, "Error importing Kibana dashboards") } diff --git a/libbeat/common/field.go b/libbeat/common/field.go index 2decdd41092..1686d1c3a78 100644 --- a/libbeat/common/field.go +++ b/libbeat/common/field.go @@ -119,7 +119,7 @@ func (f *Field) Validate() error { } func LoadFieldsYaml(path string) (Fields, error) { - keys := []Field{} + var keys []Field cfg, err := yaml.NewConfigWithFile(path) if err != nil { @@ -135,6 +135,23 @@ func LoadFieldsYaml(path string) (Fields, error) { return fields, nil } +// LoadFields loads fields from a byte array +func LoadFields(f []byte) (Fields, error) { + var keys []Field + + cfg, err := yaml.NewConfig(f) + if err != nil { + return nil, err + } + cfg.Unpack(&keys) + + fields := Fields{} + for _, key := range keys { + fields = append(fields, key.Fields...) + } + return fields, nil +} + // HasKey checks if inside fields the given key exists // The key can be in the form of a.b.c and it will check if the nested field exist // In case the key is `a` and there is a value `a.b` false is return as it only diff --git a/libbeat/dashboards/dashboards.go b/libbeat/dashboards/dashboards.go index 79d664c3db8..c1ba0ca9b1c 100644 --- a/libbeat/dashboards/dashboards.go +++ b/libbeat/dashboards/dashboards.go @@ -25,15 +25,17 @@ import ( errw "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" ) // ImportDashboards tries to import the kibana dashboards. func ImportDashboards( ctx context.Context, - beatName, hostname, homePath string, + beatInfo beat.Info, homePath string, kibanaConfig, dashboardsConfig *common.Config, msgOutputter MessageOutputter, + pattern common.MapStr, ) error { if dashboardsConfig == nil || !dashboardsConfig.Enabled() { return nil @@ -41,28 +43,22 @@ func ImportDashboards( // unpack dashboard config dashConfig := defaultConfig - dashConfig.Beat = beatName + dashConfig.Beat = beatInfo.Beat dashConfig.Dir = filepath.Join(homePath, defaultDirectory) err := dashboardsConfig.Unpack(&dashConfig) if err != nil { return err } - // init kibana config object - if kibanaConfig == nil { - kibanaConfig = common.NewConfig() - } - if !kibanaConfig.Enabled() { return errors.New("kibana configuration missing for loading dashboards.") } - return setupAndImportDashboardsViaKibana(ctx, hostname, kibanaConfig, &dashConfig, msgOutputter) - + return setupAndImportDashboardsViaKibana(ctx, beatInfo.Hostname, kibanaConfig, &dashConfig, msgOutputter, pattern) } func setupAndImportDashboardsViaKibana(ctx context.Context, hostname string, kibanaConfig *common.Config, - dashboardsConfig *Config, msgOutputter MessageOutputter) error { + dashboardsConfig *Config, msgOutputter MessageOutputter, fields common.MapStr) error { kibanaLoader, err := NewKibanaLoader(ctx, kibanaConfig, dashboardsConfig, hostname, msgOutputter) if err != nil { @@ -73,10 +69,11 @@ func setupAndImportDashboardsViaKibana(ctx context.Context, hostname string, kib kibanaLoader.statusMsg("Kibana URL %v", kibanaLoader.client.Connection.URL) - return ImportDashboardsViaKibana(kibanaLoader) + return ImportDashboardsViaKibana(kibanaLoader, fields) } -func ImportDashboardsViaKibana(kibanaLoader *KibanaLoader) error { +// ImportDashboardsViaKibana imports Dashboards to Kibana +func ImportDashboardsViaKibana(kibanaLoader *KibanaLoader, fields common.MapStr) error { version := kibanaLoader.version if !version.IsValid() { return errors.New("No valid kibana version available") @@ -86,7 +83,7 @@ func ImportDashboardsViaKibana(kibanaLoader *KibanaLoader) error { return fmt.Errorf("Kibana API is not available in Kibana version %s", kibanaLoader.version.String()) } - importer, err := NewImporter(version, kibanaLoader.config, kibanaLoader) + importer, err := NewImporter(version, kibanaLoader.config, *kibanaLoader, fields) if err != nil { return fmt.Errorf("fail to create a Kibana importer for loading the dashboards: %v", err) } diff --git a/libbeat/dashboards/importer.go b/libbeat/dashboards/importer.go index 8a1b3bc6c93..a9427651eb2 100644 --- a/libbeat/dashboards/importer.go +++ b/libbeat/dashboards/importer.go @@ -54,17 +54,12 @@ type Importer struct { cfg *Config version common.Version - loader Loader + loader KibanaLoader + fields common.MapStr } -type Loader interface { - ImportIndex(file string) error - ImportDashboard(file string) error - statusMsg(msg string, a ...interface{}) - Close() error -} - -func NewImporter(version common.Version, cfg *Config, loader Loader) (*Importer, error) { +// NewImporter creates a new dashboard importer +func NewImporter(version common.Version, cfg *Config, loader KibanaLoader, fields common.MapStr) (*Importer, error) { // Current max version is 7 if version.Major > 6 { @@ -75,6 +70,7 @@ func NewImporter(version common.Version, cfg *Config, loader Loader) (*Importer, cfg: cfg, version: version, loader: loader, + fields: fields, }, nil } @@ -106,7 +102,7 @@ func (imp Importer) ImportFile(fileType string, file string) error { if fileType == "dashboard" { return imp.loader.ImportDashboard(file) } else if fileType == "index-pattern" { - return imp.loader.ImportIndex(file) + return imp.loader.ImportIndexFile(file) } return fmt.Errorf("Unexpected file type %s", fileType) } @@ -310,6 +306,10 @@ func (imp Importer) ImportKibanaDir(dir string) error { return newErrNotFound("No directory %s", dir) } + // Loads the internal index pattern + if imp.fields != nil { + imp.loader.ImportIndex(imp.fields) + } check := []string{} if !imp.cfg.OnlyDashboards { check = append(check, "index-pattern") diff --git a/libbeat/dashboards/kibana_loader.go b/libbeat/dashboards/kibana_loader.go index 6c6e291e4ae..ca7dd345f15 100644 --- a/libbeat/dashboards/kibana_loader.go +++ b/libbeat/dashboards/kibana_loader.go @@ -40,6 +40,7 @@ type KibanaLoader struct { msgOutputter MessageOutputter } +// NewKibanaLoader creates a new loader to load Kibana files func NewKibanaLoader(ctx context.Context, cfg *common.Config, dashboardsConfig *Config, hostname string, msgOutputter MessageOutputter) (*KibanaLoader, error) { if cfg == nil || !cfg.Enabled() { @@ -81,10 +82,8 @@ func getKibanaClient(ctx context.Context, cfg *common.Config, retryCfg *Retry, r return client, nil } -func (loader KibanaLoader) ImportIndex(file string) error { - params := url.Values{} - params.Set("force", "true") //overwrite the existing dashboards - +// ImportIndexFile imports an index pattern from a file +func (loader KibanaLoader) ImportIndexFile(file string) error { // read json file reader, err := ioutil.ReadFile(file) if err != nil { @@ -97,11 +96,19 @@ func (loader KibanaLoader) ImportIndex(file string) error { return fmt.Errorf("fail to unmarshal the index content from file %s: %v", file, err) } - indexContent = ReplaceIndexInIndexPattern(loader.config.Index, indexContent) + return loader.ImportIndex(indexContent) +} + +// ImportIndex imports the passed index pattern to Kibana +func (loader KibanaLoader) ImportIndex(pattern common.MapStr) error { + params := url.Values{} + params.Set("force", "true") //overwrite the existing dashboards + indexContent := ReplaceIndexInIndexPattern(loader.config.Index, pattern) return loader.client.ImportJSON(importAPI, params, indexContent) } +// ImportDashboard imports the dashboard file func (loader KibanaLoader) ImportDashboard(file string) error { params := url.Values{} params.Set("force", "true") //overwrite the existing dashboards diff --git a/libbeat/kibana/fields_transformer.go b/libbeat/kibana/fields_transformer.go index 3ccf212c362..207c193a750 100644 --- a/libbeat/kibana/fields_transformer.go +++ b/libbeat/kibana/fields_transformer.go @@ -32,9 +32,10 @@ type fieldsTransformer struct { transformedFieldFormatMap common.MapStr version *common.Version keys map[string]int + migration bool } -func newFieldsTransformer(version *common.Version, fields common.Fields) (*fieldsTransformer, error) { +func newFieldsTransformer(version *common.Version, fields common.Fields, migration bool) (*fieldsTransformer, error) { if version == nil { return nil, errors.New("Version must be given") } @@ -44,6 +45,7 @@ func newFieldsTransformer(version *common.Version, fields common.Fields) (*field transformedFields: []common.MapStr{}, transformedFieldFormatMap: common.MapStr{}, keys: map[string]int{}, + migration: migration, }, nil } @@ -90,6 +92,10 @@ func (t *fieldsTransformer) transformFields(commonFields common.Fields, path str if t.version.LessThan(v640) { continue } + // Only adds migration aliases if migration is enabled + if f.MigrationAlias && !t.migration { + continue + } if ff := t.fields.GetField(f.AliasPath); ff != nil { // copy the field, keep path := f.Path diff --git a/libbeat/kibana/fields_transformer_test.go b/libbeat/kibana/fields_transformer_test.go index 90122667448..1fb1e8b3a0b 100644 --- a/libbeat/kibana/fields_transformer_test.go +++ b/libbeat/kibana/fields_transformer_test.go @@ -35,7 +35,7 @@ var ( ) func TestEmpty(t *testing.T) { - trans, err := newFieldsTransformer(version, common.Fields{}) + trans, err := newFieldsTransformer(version, common.Fields{}, true) assert.NoError(t, err) out, err := trans.transform() assert.NoError(t, err) @@ -93,7 +93,7 @@ func TestEmpty(t *testing.T) { func TestMissingVersion(t *testing.T) { var c *common.Version - _, err := newFieldsTransformer(c, common.Fields{}) + _, err := newFieldsTransformer(c, common.Fields{}, true) assert.Error(t, err) } @@ -118,9 +118,10 @@ func TestDuplicateField(t *testing.T) { }}, } for _, testCase := range testCases { - trans, err := newFieldsTransformer(version, testCase.commonFields) + trans, err := newFieldsTransformer(version, testCase.commonFields, true) require.NoError(t, err) _, err = trans.transform() + fmt.Println(err) assert.Error(t, err) } } @@ -151,7 +152,7 @@ func TestValidDuplicateField(t *testing.T) { }, }, } - trans, err := newFieldsTransformer(version, commonFields) + trans, err := newFieldsTransformer(version, commonFields, true) require.NoError(t, err) transformed, err := trans.transform() require.NoError(t, err) @@ -179,7 +180,7 @@ func TestInvalidVersion(t *testing.T) { }, }, } - trans, err := newFieldsTransformer(version, commonFields) + trans, err := newFieldsTransformer(version, commonFields, true) assert.NoError(t, err) _, err = trans.transform() assert.Error(t, err) @@ -206,7 +207,7 @@ func TestTransformTypes(t *testing.T) { {commonField: common.Field{Type: "invalid"}, expected: nil}, } for idx, test := range tests { - trans, _ := newFieldsTransformer(version, common.Fields{test.commonField}) + trans, _ := newFieldsTransformer(version, common.Fields{test.commonField}, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fields"].([]common.MapStr)[0] @@ -251,7 +252,7 @@ func TestTransformGroup(t *testing.T) { }, } for idx, test := range tests { - trans, _ := newFieldsTransformer(version, test.commonFields) + trans, _ := newFieldsTransformer(version, test.commonFields, false) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fields"].([]common.MapStr) @@ -327,7 +328,7 @@ func TestTransformMisc(t *testing.T) { {commonField: common.Field{Script: "doc[]"}, expected: "painless", attr: "lang"}, } for idx, test := range tests { - trans, _ := newFieldsTransformer(version, common.Fields{test.commonField}) + trans, _ := newFieldsTransformer(version, common.Fields{test.commonField}, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fields"].([]common.MapStr)[0] @@ -526,7 +527,7 @@ func TestTransformFieldFormatMap(t *testing.T) { }, } for idx, test := range tests { - trans, _ := newFieldsTransformer(test.version, common.Fields{test.commonField}) + trans, _ := newFieldsTransformer(test.version, common.Fields{test.commonField}, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fieldFormatMap"] @@ -594,7 +595,7 @@ func TestTransformGroupAndEnabled(t *testing.T) { }, } for idx, test := range tests { - trans, _ := newFieldsTransformer(version, test.commonFields) + trans, _ := newFieldsTransformer(version, test.commonFields, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fields"].([]common.MapStr) @@ -614,7 +615,7 @@ func TestTransformMultiField(t *testing.T) { common.Field{Name: "text", Type: "text"}, }, } - trans, _ := newFieldsTransformer(version, common.Fields{f}) + trans, _ := newFieldsTransformer(version, common.Fields{f}, true) transformed, err := trans.transform() assert.NoError(t, err) out := transformed["fields"].([]common.MapStr) diff --git a/libbeat/kibana/index_pattern_generator.go b/libbeat/kibana/index_pattern_generator.go index d96646e7ad6..8252177d27f 100644 --- a/libbeat/kibana/index_pattern_generator.go +++ b/libbeat/kibana/index_pattern_generator.go @@ -28,45 +28,34 @@ import ( ) type IndexPatternGenerator struct { - indexName string - beatVersion string - fieldsYaml string - version common.Version - targetDir string - targetFilename string + indexName string + beatVersion string + fields []byte + version common.Version + migration bool } // Create an instance of the Kibana Index Pattern Generator -func NewGenerator(indexName, beatName, fieldsYAMLFile, outputDir, beatVersion string, version common.Version) (*IndexPatternGenerator, error) { +func NewGenerator(indexName, beatName string, fields []byte, beatVersion string, version common.Version, migration bool) (*IndexPatternGenerator, error) { beatName = clean(beatName) - if _, err := os.Stat(fieldsYAMLFile); err != nil { - return nil, err - } - return &IndexPatternGenerator{ - indexName: indexName, - fieldsYaml: fieldsYAMLFile, - beatVersion: beatVersion, - version: version, - targetDir: createTargetDir(outputDir, version), - targetFilename: beatName + ".json", + indexName: indexName + "-*", + fields: fields, + beatVersion: beatVersion, + version: version, + migration: migration, }, nil } // Generate creates the Index-Pattern for Kibana. -func (i *IndexPatternGenerator) Generate() (string, error) { +func (i *IndexPatternGenerator) Generate() (common.MapStr, error) { idxPattern, err := i.generate() if err != nil { - return "", err + return nil, err } - idxPattern = i.generatePattern(idxPattern) - - file := filepath.Join(i.targetDir, i.targetFilename) - err = dumpToFile(file, idxPattern) - - return file, err + return i.generatePattern(idxPattern), nil } func (i *IndexPatternGenerator) generate() (common.MapStr, error) { @@ -105,7 +94,7 @@ func (i *IndexPatternGenerator) generatePattern(attrs common.MapStr) common.MapS } func (i *IndexPatternGenerator) addGeneral(indexPattern *common.MapStr) error { - kibanaEntries, err := loadKibanaEntriesFromYaml(i.fieldsYaml) + kibanaEntries, err := loadKibanaEntriesFromYaml(i.fields) if err != nil { return err } @@ -121,11 +110,11 @@ func (i *IndexPatternGenerator) addGeneral(indexPattern *common.MapStr) error { } func (i *IndexPatternGenerator) addFieldsSpecific(indexPattern *common.MapStr) error { - fields, err := common.LoadFieldsYaml(i.fieldsYaml) + fields, err := common.LoadFields(i.fields) if err != nil { return err } - transformer, err := newFieldsTransformer(&i.version, fields) + transformer, err := newFieldsTransformer(&i.version, fields, i.migration) if err != nil { return err } diff --git a/libbeat/kibana/index_pattern_generator_test.go b/libbeat/kibana/index_pattern_generator_test.go index 302eaafb693..38cdc2dc983 100644 --- a/libbeat/kibana/index_pattern_generator_test.go +++ b/libbeat/kibana/index_pattern_generator_test.go @@ -22,7 +22,6 @@ import ( "fmt" "io/ioutil" "os" - "path/filepath" "strings" "testing" @@ -39,27 +38,20 @@ func TestNewGenerator(t *testing.T) { tmpDir := tmpPath(t) defer os.RemoveAll(tmpDir) - v, _ := common.NewVersion("7.0.0") - // checks for fields.yml - generator, err := NewGenerator("beat-index", "mybeat.", fieldsYml+".missing", tmpDir, "7.0", *v) - assert.Error(t, err) - - generator, err = NewGenerator("beat-index", "mybeat.", fieldsYml, tmpDir, "7.0", *v) + data, err := ioutil.ReadFile("./testdata/fields.yml") if err != nil { t.Fatal(err) } - assert.Equal(t, "7.0", generator.beatVersion) - assert.Equal(t, "beat-index", generator.indexName) - // creates file dir and sets name - expectedDir := filepath.Join(tmpDir, "7/index-pattern") - assert.Equal(t, expectedDir, generator.targetDir) - _, err = os.Stat(generator.targetDir) + v, _ := common.NewVersion("7.0.0") + // checks for fields.yml + generator, err := NewGenerator("beat-index", "mybeat.", data, "7.0", *v, true) if err != nil { t.Fatal(err) } + assert.Equal(t, "7.0", generator.beatVersion) + assert.Equal(t, "beat-index-*", generator.indexName) - assert.Equal(t, "mybeat.json", generator.targetFilename) } func TestCleanName(t *testing.T) { @@ -78,129 +70,102 @@ func TestCleanName(t *testing.T) { } } -func TestGenerateFieldsYaml(t *testing.T) { - tmpDir := tmpPath(t) - defer os.RemoveAll(tmpDir) - - v, _ := common.NewVersion("6.0.0") - generator, err := NewGenerator("metricbeat-*", "metric beat ?!", fieldsYml, tmpDir, "7.0.0-alpha1", *v) - if err != nil { - t.Fatal(err) - } - - _, err = generator.Generate() - if err != nil { - t.Fatal(err) - } - - generator.fieldsYaml = "" - _, err = generator.Generate() - assert.Error(t, err) -} - -func TestDumpToFileDefault(t *testing.T) { - tmpDir := tmpPath(t) - defer os.RemoveAll(tmpDir) - - v, _ := common.NewVersion("7.0.0") - generator, err := NewGenerator("metricbeat-*", "metric beat ?!", fieldsYml, tmpDir, "7.0.0-alpha1", *v) - if err != nil { - t.Fatal(err) - } - - _, err = generator.Generate() - if err != nil { - t.Fatal(err) - } - - generator.targetDir = filepath.Join(tmpDir, "./non-existing/something") - _, err = generator.Generate() - assert.Error(t, err) -} - func TestGenerate(t *testing.T) { tmpDir := tmpPath(t) defer os.RemoveAll(tmpDir) - v6, _ := common.NewVersion("6.4.0") - versions := []*common.Version{v6} + v7, _ := common.NewVersion("7.0.0-alpha1") + versions := []*common.Version{v7} + var d common.MapStr for _, version := range versions { - generator, err := NewGenerator("beat-*", "b eat ?!", fieldsYml, tmpDir, "7.0.0-alpha1", *version) + data, err := ioutil.ReadFile("./testdata/fields.yml") + if err != nil { + t.Fatal(err) + } + generator, err := NewGenerator("beat", "b eat ?!", data, version.String(), *version, true) if err != nil { t.Fatal(err) } - _, err = generator.Generate() + d, err = generator.Generate() if err != nil { t.Fatal(err) } } - tests := []map[string]string{ + tests := []compare{ { - "existing": "testdata/beat-6.json", - "created": filepath.Join(tmpDir, "7/index-pattern/beat.json"), + existing: "testdata/beat-6.json", + created: d, }, } + testGenerate(t, tests, true) } +type compare struct { + existing string + created common.MapStr +} + func TestGenerateExtensive(t *testing.T) { tmpDir := tmpPath(t) defer os.RemoveAll(tmpDir) - version6, _ := common.NewVersion("6.4.0") - versions := []*common.Version{version6} + version7, _ := common.NewVersion("7.0.0-alpha1") + versions := []*common.Version{version7} + + var d common.MapStr for _, version := range versions { - generator, err := NewGenerator("metricbeat-*", "metric be at ?!", "testdata/extensive/fields.yml", tmpDir, "7.0.0-alpha1", *version) + data, err := ioutil.ReadFile("testdata/extensive/fields.yml") + if err != nil { + t.Fatal(err) + } + generator, err := NewGenerator("metricbeat", "metric be at ?!", data, version.String(), *version, true) if err != nil { t.Fatal(err) } - _, err = generator.Generate() + d, err = generator.Generate() if err != nil { t.Fatal(err) } } - tests := []map[string]string{ + tests := []compare{ { - "existing": "testdata/extensive/metricbeat-6.json", - "created": filepath.Join(tmpDir, "7/index-pattern/metricbeat.json"), + existing: "testdata/extensive/metricbeat-6.json", + created: d, }, } testGenerate(t, tests, false) } -func testGenerate(t *testing.T, tests []map[string]string, sourceFilters bool) { +func testGenerate(t *testing.T, tests []compare, sourceFilters bool) { for _, test := range tests { // compare default - existing, err := readJson(test["existing"]) - if err != nil { - t.Fatal(err) - } - created, err := readJson(test["created"]) + existing, err := readJson(test.existing) if err != nil { t.Fatal(err) } var attrExisting, attrCreated common.MapStr - if strings.Contains(test["existing"], "6") { - assert.Equal(t, existing["version"], created["version"]) + if strings.Contains(test.existing, "6") { + assert.Equal(t, existing["version"], test.created["version"]) objExisting := existing["objects"].([]interface{})[0].(map[string]interface{}) - objCreated := created["objects"].([]interface{})[0].(map[string]interface{}) + objCreated := test.created["objects"].([]common.MapStr)[0] - assert.Equal(t, objExisting["version"], objCreated["version"]) + assert.Equal(t, int(objExisting["version"].(float64)), objCreated["version"]) assert.Equal(t, objExisting["id"], objCreated["id"]) assert.Equal(t, objExisting["type"], objCreated["type"]) attrExisting = objExisting["attributes"].(map[string]interface{}) - attrCreated = objCreated["attributes"].(map[string]interface{}) + attrCreated = objCreated["attributes"].(common.MapStr) } else { attrExisting = existing - attrCreated = created + attrCreated = test.created } // check fieldFormatMap diff --git a/libbeat/kibana/transformer.go b/libbeat/kibana/transformer.go index 9c4649e2cec..f99dd5ff0c0 100644 --- a/libbeat/kibana/transformer.go +++ b/libbeat/kibana/transformer.go @@ -50,9 +50,9 @@ func (t *transformer) transform() common.MapStr { return transformed } -func loadKibanaEntriesFromYaml(yamlFile string) ([]kibanaEntry, error) { +func loadKibanaEntriesFromYaml(fields []byte) ([]kibanaEntry, error) { entries := []kibanaEntry{} - cfg, err := yaml.NewConfigWithFile(yamlFile) + cfg, err := yaml.NewConfig(fields) if err != nil { return nil, err } diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 47196dd5f9c..6beccd87139 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -360,9 +360,6 @@ endif fi @# Convert all dashboards to string @python ${ES_BEATS}/libbeat/scripts/unpack_dashboards.py --glob="./_meta/kibana.generated/7/dashboard/*.json" - @go run ${ES_BEATS}/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go \ - -index '${BEAT_INDEX_PREFIX}-*' -beat ${BEAT_NAME} -fields $(PWD)/fields.yml \ - -version ${BEAT_VERSION} -out _meta/kibana.generated endif .PHONY: docs diff --git a/libbeat/tests/system/test_index_pattern.py b/libbeat/tests/system/test_index_pattern.py new file mode 100644 index 00000000000..85f86684e7b --- /dev/null +++ b/libbeat/tests/system/test_index_pattern.py @@ -0,0 +1,21 @@ +from base import BaseTest +import os + + +class Test(BaseTest): + + def test_export_index_pattern(self): + """ + Test export index pattern + """ + self.render_config_template("mockbeat", + os.path.join(self.working_dir, + "mockbeat.yml"), + fields=os.path.join(self.working_dir, "fields.yml")) + exit_code = self.run_beat( + logging_args=[], + extra_args=["export", "index-pattern"], + config="mockbeat.yml") + + assert exit_code == 0 + assert self.log_contains('"objects": [') diff --git a/metricbeat/Makefile b/metricbeat/Makefile index a034e954b44..710abbe9f89 100644 --- a/metricbeat/Makefile +++ b/metricbeat/Makefile @@ -78,5 +78,4 @@ test-module: python-env update metricbeat.test assets: go run ${ES_BEATS}/metricbeat/scripts/assets/assets.go ${ES_BEATS}/metricbeat/module mkdir -p include/fields - go run ${ES_BEATS}/dev-tools/cmd/asset/asset.go -license ${LICENSE} -pkg include -in ${ES_BEATS}/metricbeat/_meta/fields.common.yml -out include/fields.go $(BEAT_NAME) go run ${ES_BEATS}/libbeat/scripts/cmd/global_fields/main.go -es_beats_path ${ES_BEATS} -beat_path ${PWD} | go run ${ES_BEATS}/dev-tools/cmd/asset/asset.go -license ${LICENSE} -out ./include/fields/fields.go -pkg include -priority asset.LibbeatFieldsPri ${ES_BEATS}/libbeat/fields.yml $(BEAT_NAME) diff --git a/metricbeat/include/fields.go b/metricbeat/include/fields.go deleted file mode 100644 index a1115095c3c..00000000000 --- a/metricbeat/include/fields.go +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. - -package include - -import ( - "github.com/elastic/beats/libbeat/asset" -) - -func init() { - if err := asset.SetFields("metricbeat", "../metricbeat/_meta/fields.common.yml", asset.BeatFieldsPri, AssetMetricbeatMetaFieldsCommonYml); err != nil { - panic(err) - } -} - -// AssetMetricbeatMetaFieldsCommonYml returns asset data. -// This is the base64 encoded gzipped contents of ../metricbeat/_meta/fields.common.yml. -func AssetMetricbeatMetaFieldsCommonYml() string { - return "eJyUU0tu3DAM3fsUD9nHB/CiQJBNl13kAozEWET1cUV6pr59YVkT1EUzab0TxfcR+fyI77xNcCWlkgfAxCJPeL6dPaurspiUPOHLAADPJRtJ1g7Cm3D0CrqQRHqNDMmgGMEXzgbbFtZxQG+bhsbxiEyJJyS2Kk7ZxlT8Grld/lV1/14CNxzKGywwDgwskGHmzJWMfbtp2mMH7hYmUBTSXlnIwtSbTsJJ5kqHrNWVPzK7n//T6g17x+1JbKnFseq4zOJP74glz/e1vx1QzLWsC8T/waxcL+J4JO8rq97nejqa3p9BLkhmXANXbpXOBlHUNWfJ84iXIPrO0PaORBtyMbwylsq6J+MaODcKT0a40h6oGNntU4nFUYzbB85DUft8BV+L2nkH/2L+rLiPvBNW/rFKZd+TcRT5J6Ul/p6NT+a558IXt6bbvzHiKV5pU7RsFDz44h7G4VcAAAD//8J8HrM=" -}