Disable migration aliases in index pattern

CHANGELOG-developer.next.asciidoc

@@ -21,6 +21,7 @@ The list below covers the major changes between 7.0.0-alpha2 and master only.
 ==== Breaking changes
 - Outputs receive Index Manager as additional parameter. The index manager can
   be used to create an index selector. {pull}10347[10347]
+- Remove support for loading dashboards to Elasticsearch 5. {pull}10451[10451]
 
 ==== Bugfixes
 
@@ -31,3 +32,4 @@ The list below covers the major changes between 7.0.0-alpha2 and master only.
 - Add (*common.Config).Has and (*common.Config).Remove. {pull}10363[10363]
 - Introduce ILM and IndexManagment support to beat.Settings. {pull}10347[10347]
 - Introduce ILM and IndexManagement support to beat.Settings. {pull}10347[10347]
+- Generating index pattern on demand instead of shipping them in the packages. {pull}10478[10478]

CHANGELOG.next.asciidoc

@@ -203,6 +203,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add ILM mode `auto` to setup.ilm.enabled setting. This new default value detects if ILM is available {pull}10347[10347]
 - Add support to read ILM policy from external JSON file. {pull}10347[10347]
 - Add `overwrite` and `check_exists` settings to ILM support. {pull}10347[10347]
+- Generate Kibana index pattern on demand instead of using a local file. {pull}10478[10478]
 
 *Auditbeat*
 

dev-tools/mage/kibana.go

@@ -74,22 +74,7 @@ func KibanaDashboards(moduleDirs ...string) error {
 		return err
 	}
 
-	beatVersion, err := BeatQualifiedVersion()
-	if err != nil {
-		return err
-	}
-
-	// Generate Kibana index pattern files from fields.yml.
-	indexPatternCmd := sh.RunCmd("go", "run",
-		filepath.Join(esBeatsDir, "dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go"),
-		"-beat", BeatName,
-		"-version", beatVersion,
-		"-index", BeatIndexPrefix+"-*",
-		"-fields", "fields.yml",
-		"-out", kibanaBuildDir,
-	)
-
-	return indexPatternCmd()
+	return nil
 }
 
 // PackageKibanaDashboardsFromBuildDir reconfigures the packaging configuration

filebeat/tests/system/test_index_pattern.py

@@ -0,0 +1,32 @@
+import os
+import unittest
+from filebeat import BaseTest
+
+
+class Test(BaseTest):
+
+    def test_export_index_pattern(self):
+        """
+        Test export index pattern
+        """
+        self.render_config_template()
+        exit_code = self.run_beat(
+            logging_args=[],
+            extra_args=["export", "index-pattern"])
+
+        assert exit_code == 0
+        assert self.log_contains('"objects": [')
+        assert self.log_contains('beat.name') == False
+
+    def test_export_index_pattern_migration(self):
+        """
+        Test export index pattern with migration flag enabled
+        """
+        self.render_config_template()
+        exit_code = self.run_beat(
+            logging_args=[],
+            extra_args=["export", "index-pattern", "-E", "migration.enabled:true"])
+
+        assert exit_code == 0
+        assert self.log_contains('"objects": [')
+        assert self.log_contains('beat.name')

libbeat/cmd/export.go

@@ -32,6 +32,7 @@ func genExportCmd(settings instance.Settings, name, idxPrefix, beatVersion strin
 
 	exportCmd.AddCommand(export.GenExportConfigCmd(settings, name, idxPrefix, beatVersion))
 	exportCmd.AddCommand(export.GenTemplateConfigCmd(settings, name, idxPrefix, beatVersion))
+	exportCmd.AddCommand(export.GenIndexPatternConfigCmd(settings, name, idxPrefix, beatVersion))
 	exportCmd.AddCommand(export.GenDashboardCmd(name, idxPrefix, beatVersion))
 	exportCmd.AddCommand(export.GenGetILMPolicyCmd(settings, name, idxPrefix, beatVersion))
 

libbeat/cmd/export/index_pattern.go

@@ -0,0 +1,87 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package export
+
+import (
+	"log"
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"github.com/elastic/beats/libbeat/cmd/instance"
+	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/kibana"
+)
+
+// GenIndexPatternConfigCmd generates an index pattern for Kibana
+func GenIndexPatternConfigCmd(settings instance.Settings, name, idxPrefix, beatVersion string) *cobra.Command {
+	genTemplateConfigCmd := &cobra.Command{
+		Use:   "index-pattern",
+		Short: "Export kibana index pattern to stdout",
+		Run: func(cmd *cobra.Command, args []string) {
+			version, _ := cmd.Flags().GetString("es.version")
+
+			b, err := instance.NewBeat(name, idxPrefix, beatVersion)
+			if err != nil {
+				fatalf("Error initializing beat: %+v", err)
+			}
+			err = b.InitWithSettings(settings)
+			if err != nil {
+				fatalf("Error initializing beat: %+v", err)
+			}
+
+			if version == "" {
+				version = b.Info.Version
+			}
+
+			var withMigration bool
+			if b.RawConfig.HasField("migration") {
+				sub, err := b.RawConfig.Child("migration", -1)
+				if err != nil {
+					fatalf("Failed to read migration setting: %+v", err)
+				}
+				withMigration = sub.Enabled()
+			}
+
+			// Index pattern generation
+			v, err := common.NewVersion(version)
+			if err != nil {
+				fatalf("Error creating version: %+v", err)
+			}
+			indexPattern, err := kibana.NewGenerator(b.Info.IndexPrefix, b.Info.Beat, b.Fields, beatVersion, *v, withMigration)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			pattern, err := indexPattern.Generate()
+			if err != nil {
+				log.Fatalf("ERROR: %s", err)
+			}
+
+			_, err = os.Stdout.WriteString(pattern.StringToPrint() + "\n")
+			if err != nil {
+				fatalf("Error writing index pattern: %+v", err)
+			}
+		},
+	}
+
+	genTemplateConfigCmd.Flags().String("es.version", beatVersion, "Elasticsearch version")
+	genTemplateConfigCmd.Flags().String("index", idxPrefix, "Base index name")
+
+	return genTemplateConfigCmd
+}

libbeat/cmd/instance/beat.go

@@ -32,6 +32,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/elastic/beats/libbeat/kibana"
+
 	"github.com/gofrs/uuid"
 	errw "github.com/pkg/errors"
 	"go.uber.org/zap"
@@ -679,8 +681,42 @@ func (b *Beat) loadDashboards(ctx context.Context, force bool) error {
 	}
 
 	if b.Config.Dashboards.Enabled() {
-		err := dashboards.ImportDashboards(ctx, b.Info.Beat, b.Info.Hostname, paths.Resolve(paths.Home, ""),
-			b.Config.Kibana, b.Config.Dashboards, nil)
+
+		var withMigration bool
+		if b.RawConfig.HasField("migration") {
+			sub, err := b.RawConfig.Child("migration", -1)
+			if err != nil {
+				return fmt.Errorf("Failed to read migration setting: %+v", err)
+			}
+			withMigration = sub.Enabled()
+		}
+
+		// init kibana config object
+		kibanaConfig := b.Config.Kibana
+		if kibanaConfig == nil {
+			kibanaConfig = common.NewConfig()
+		}
+
+		client, err := kibana.NewKibanaClient(kibanaConfig)
+		if err != nil {
+			return fmt.Errorf("error connecting to Kibana: %v", err)
+		}
+		// This fetches the version for Kibana. For the alias feature the version of ES would be needed
+		// but it's assumed that KB and ES have the same minor version.
+		v := client.GetVersion()
+
+		indexPattern, err := kibana.NewGenerator(b.Info.IndexPrefix, b.Info.Beat, b.Fields, b.Info.Version, v, withMigration)
+		if err != nil {
+			return fmt.Errorf("error creating index pattern generator: %v", err)
+		}
+
+		pattern, err := indexPattern.Generate()
+		if err != nil {
+			return fmt.Errorf("error generating index pattern: %v", err)
+		}
+
+		err = dashboards.ImportDashboards(ctx, b.Info, paths.Resolve(paths.Home, ""),
+			kibanaConfig, b.Config.Dashboards, nil, pattern)
 		if err != nil {
 			return errw.Wrap(err, "Error importing Kibana dashboards")
 		}

libbeat/common/field.go

@@ -119,7 +119,7 @@ func (f *Field) Validate() error {
 }
 
 func LoadFieldsYaml(path string) (Fields, error) {
-	keys := []Field{}
+	var keys []Field
 
 	cfg, err := yaml.NewConfigWithFile(path)
 	if err != nil {
@@ -135,6 +135,23 @@ func LoadFieldsYaml(path string) (Fields, error) {
 	return fields, nil
 }
 
+// LoadFields loads fields from a byte array
+func LoadFields(f []byte) (Fields, error) {
+	var keys []Field
+
+	cfg, err := yaml.NewConfig(f)
+	if err != nil {
+		return nil, err
+	}
+	cfg.Unpack(&keys)
+
+	fields := Fields{}
+	for _, key := range keys {
+		fields = append(fields, key.Fields...)
+	}
+	return fields, nil
+}
+
 // HasKey checks if inside fields the given key exists
 // The key can be in the form of a.b.c and it will check if the nested field exist
 // In case the key is `a` and there is a value `a.b` false is return as it only