From 3c16658ee07e92b9271d2c42f93add85949eb3ae Mon Sep 17 00:00:00 2001 From: Dylan Guedes Date: Fri, 29 Oct 2021 03:06:21 -0300 Subject: [PATCH] Loki: Append loopback to ingester net interface default list (#4570) * Move LoopbackInterfaceName to its own package. - Create new package `util/net` and puts `LoopbackInterfaceName` there - Modify distributor_test to use `util/net` package * By default, append loopback to ingester interfaces. - Append the loopback net interface to the list of net interfaces used by the ingester ring. The append only occurs if the ingester net interfaces are default - Add CHANGELOG entry * Test ring net interface behavior. - Add tests to check the current net interface name resolution. The tests checks if the ingester ring net interface names are correctly copied to other rings --- CHANGELOG.md | 3 +- pkg/distributor/distributor_test.go | 21 +------- pkg/loki/config_wrapper.go | 18 ++++++- pkg/loki/config_wrapper_test.go | 79 +++++++++++++++++++++++++++++ pkg/util/net/net_interfaces.go | 23 +++++++++ 5 files changed, 122 insertions(+), 22 deletions(-) create mode 100644 pkg/util/net/net_interfaces.go diff --git a/CHANGELOG.md b/CHANGELOG.md index a39d60568a4c..7b6d96a8e93c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,10 @@ * [4519](https://github.com/grafana/loki/pull/4519) **DylanGuedes** and **replay**: Loki: Enable FIFO cache by default * [4520](https://github.com/grafana/loki/pull/4520) **jordanrushing** and **owen-d**: Introduce overrides-exporter module for tenant limits * [4453](https://github.com/grafana/loki/pull/4453) **liguozhong**: Loki: Implement retry to s3 chunk storage -* [4542](https://github.com/grafana/loki/pull/4542) **owen-d**: Introduce the `loki_overrides_defaults` metric and only export diffs for tenant limits. +* [4542](https://github.com/grafana/loki/pull/4542) **owen-d**: Introduce the `loki_overrides_defaults` metric and only export diffs for tenant limits * [4498](https://github.com/grafana/loki/pull/4498) **trevorwhitney**: Feature: add virtual read and write targets * [4543](https://github.com/grafana/loki/pull/4543) **trevorwhitney**: Change more default values and improve application of common storage config +* [4570](https://github.com/grafana/loki/pull/4570) **DylanGuedes**: Loki: Append loopback to ingester net interface default list # 2.3.0 (2021/08/06) diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go index 8faf603d9782..147ae5dbf435 100644 --- a/pkg/distributor/distributor_test.go +++ b/pkg/distributor/distributor_test.go @@ -5,7 +5,6 @@ import ( "fmt" "math" "math/rand" - "net" "net/http" "strconv" "strings" @@ -33,6 +32,7 @@ import ( "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/runtime" fe "github.com/grafana/loki/pkg/util/flagext" + loki_net "github.com/grafana/loki/pkg/util/net" "github.com/grafana/loki/pkg/validation" ) @@ -308,23 +308,6 @@ func TestDistributor_PushIngestionRateLimiter(t *testing.T) { } } -// loopbackInterfaceName search for the name of a loopback interface in the list -// of the system's network interfaces. -func loopbackInterfaceName() (string, error) { - is, err := net.Interfaces() - if err != nil { - return "", fmt.Errorf("can't retrieve loopback interface name: %s", err) - } - - for _, i := range is { - if i.Flags&net.FlagLoopback != 0 { - return i.Name, nil - } - } - - return "", fmt.Errorf("can't retrieve loopback interface name") -} - func prepare(t *testing.T, limits *validation.Limits, kvStore kv.Client, factory func(addr string) (ring_client.PoolClient, error)) *Distributor { var ( distributorConfig Config @@ -350,7 +333,7 @@ func prepare(t *testing.T, limits *validation.Limits, kvStore kv.Client, factory }) } - loopbackName, err := loopbackInterfaceName() + loopbackName, err := loki_net.LoopbackInterfaceName() require.NoError(t, err) distributorConfig.DistributorRing.HeartbeatPeriod = 100 * time.Millisecond diff --git a/pkg/loki/config_wrapper.go b/pkg/loki/config_wrapper.go index ba392712b00d..5522fe686de3 100644 --- a/pkg/loki/config_wrapper.go +++ b/pkg/loki/config_wrapper.go @@ -15,6 +15,7 @@ import ( loki_storage "github.com/grafana/loki/pkg/storage" chunk_storage "github.com/grafana/loki/pkg/storage/chunk/storage" + loki_net "github.com/grafana/loki/pkg/util/net" ) // ConfigWrapper is a struct containing the Loki config along with other values that can be set on the command line @@ -75,8 +76,8 @@ func (c *ConfigWrapper) ApplyDynamicConfig() cfg.Source { r.QueryScheduler.UseSchedulerRing = true } - applyIngesterRingConfig(r) applyPathPrefixDefaults(r, defaults) + applyIngesterRingConfig(r, &defaults) applyMemberlistConfig(r) if err := applyStorageConfig(r, &defaults); err != nil { @@ -97,7 +98,14 @@ func (c *ConfigWrapper) ApplyDynamicConfig() cfg.Source { // we have a ring configured. The reason for centralizing on the ingester ring as this is been set in basically // all of our provided config files for all of time, usually set to `inmemory` for all the single binary Loki's // and is the most central ring config for Loki. -func applyIngesterRingConfig(r *ConfigWrapper) { +// +// If the ingester ring has its interface names sets to a value equal to the default (["eth0", en0"]), it will try to append +// the loopback interface at the end of it. +func applyIngesterRingConfig(r *ConfigWrapper, defaults *ConfigWrapper) { + if reflect.DeepEqual(r.Ingester.LifecyclerConfig.InfNames, defaults.Ingester.LifecyclerConfig.InfNames) { + appendLoopbackInterface(r) + } + lc := r.Ingester.LifecyclerConfig rc := r.Ingester.LifecyclerConfig.RingConfig s := rc.KVStore.Store @@ -158,6 +166,12 @@ func applyPathPrefixDefaults(r *ConfigWrapper, defaults ConfigWrapper) { } } +func appendLoopbackInterface(r *ConfigWrapper) { + if loopbackIface, err := loki_net.LoopbackInterfaceName(); err == nil { + r.Ingester.LifecyclerConfig.InfNames = append(r.Ingester.LifecyclerConfig.InfNames, loopbackIface) + } +} + // applyMemberlistConfig will change the default ingester, distributor, ruler, and query scheduler ring configurations to use memberlist // if the -memberlist.join_members config is provided. The idea here is that if a user explicitly configured the // memberlist configuration section, they probably want to be using memberlist for all their ring configurations. diff --git a/pkg/loki/config_wrapper_test.go b/pkg/loki/config_wrapper_test.go index 9376bd34e836..46e480fa5a2c 100644 --- a/pkg/loki/config_wrapper_test.go +++ b/pkg/loki/config_wrapper_test.go @@ -22,6 +22,7 @@ import ( "github.com/grafana/loki/pkg/storage/chunk/storage" "github.com/grafana/loki/pkg/util/cfg" + loki_net "github.com/grafana/loki/pkg/util/net" ) func configWrapperFromYAML(t *testing.T, configFileString string, args []string) (ConfigWrapper, ConfigWrapper, error) { @@ -914,3 +915,81 @@ func Test_applyIngesterRingConfig(t *testing.T) { assert.Equal(t, 8, reflect.TypeOf(distributor.RingConfig{}).NumField(), fmt.Sprintf(msgf, reflect.TypeOf(distributor.RingConfig{}).String())) } + +func TestRingInterfaceNames(t *testing.T) { + defaultIface, err := loki_net.LoopbackInterfaceName() + assert.NoError(t, err) + assert.NotEmpty(t, defaultIface) + + t.Run("by default, loopback is available for all ring interfaces", func(t *testing.T) { + config, _, err := configWrapperFromYAML(t, minimalConfig, []string{}) + assert.NoError(t, err) + + assert.Contains(t, config.Ingester.LifecyclerConfig.InfNames, defaultIface) + assert.Contains(t, config.Distributor.DistributorRing.InstanceInterfaceNames, defaultIface) + assert.Contains(t, config.QueryScheduler.SchedulerRing.InstanceInterfaceNames, defaultIface) + assert.Contains(t, config.Ruler.Ring.InstanceInterfaceNames, defaultIface) + }) + + t.Run("if ingestor interface is set, it overrides other rings default interfaces", func(t *testing.T) { + yamlContent := `ingester: + lifecycler: + interface_names: + - ingesteriface` + + config, _, err := configWrapperFromYAML(t, yamlContent, []string{}) + assert.NoError(t, err) + assert.Equal(t, config.Distributor.DistributorRing.InstanceInterfaceNames, []string{"ingesteriface"}) + assert.Equal(t, config.QueryScheduler.SchedulerRing.InstanceInterfaceNames, []string{"ingesteriface"}) + assert.Equal(t, config.Ruler.Ring.InstanceInterfaceNames, []string{"ingesteriface"}) + assert.Equal(t, config.Ingester.LifecyclerConfig.InfNames, []string{"ingesteriface"}) + }) + + t.Run("if all rings have different net interface sets, doesn't override any of them", func(t *testing.T) { + yamlContent := `distributor: + ring: + instance_interface_names: + - distributoriface +ruler: + ring: + instance_interface_names: + - ruleriface +query_scheduler: + scheduler_ring: + instance_interface_names: + - scheduleriface +ingester: + lifecycler: + interface_names: + - ingesteriface` + + config, _, err := configWrapperFromYAML(t, yamlContent, []string{}) + assert.NoError(t, err) + assert.Equal(t, config.Ingester.LifecyclerConfig.InfNames, []string{"ingesteriface"}) + assert.Equal(t, config.Distributor.DistributorRing.InstanceInterfaceNames, []string{"distributoriface"}) + assert.Equal(t, config.QueryScheduler.SchedulerRing.InstanceInterfaceNames, []string{"scheduleriface"}) + assert.Equal(t, config.Ruler.Ring.InstanceInterfaceNames, []string{"ruleriface"}) + }) + + t.Run("if all rings except ingester have net interface sets, doesn't override them with ingester default value", func(t *testing.T) { + yamlContent := `distributor: + ring: + instance_interface_names: + - distributoriface +ruler: + ring: + instance_interface_names: + - ruleriface +query_scheduler: + scheduler_ring: + instance_interface_names: + - scheduleriface` + + config, _, err := configWrapperFromYAML(t, yamlContent, []string{}) + assert.NoError(t, err) + assert.Equal(t, config.Distributor.DistributorRing.InstanceInterfaceNames, []string{"distributoriface"}) + assert.Equal(t, config.QueryScheduler.SchedulerRing.InstanceInterfaceNames, []string{"scheduleriface"}) + assert.Equal(t, config.Ruler.Ring.InstanceInterfaceNames, []string{"ruleriface"}) + assert.Equal(t, config.Ingester.LifecyclerConfig.InfNames, []string{"eth0", "en0", defaultIface}) + }) +} diff --git a/pkg/util/net/net_interfaces.go b/pkg/util/net/net_interfaces.go new file mode 100644 index 000000000000..b571361c3c9e --- /dev/null +++ b/pkg/util/net/net_interfaces.go @@ -0,0 +1,23 @@ +package net + +import ( + "fmt" + "net" +) + +// LoopbackInterfaceName search for the name of a loopback interface in the list +// of the system's network interfaces and returns the first one found. +func LoopbackInterfaceName() (string, error) { + is, err := net.Interfaces() + if err != nil { + return "", fmt.Errorf("can't retrieve loopback interface name: %s", err) + } + + for _, i := range is { + if i.Flags&net.FlagLoopback != 0 { + return i.Name, nil + } + } + + return "", fmt.Errorf("can't retrieve loopback interface name") +}