Loki: Append loopback to ingester net interface default list (#4570)

* Move LoopbackInterfaceName to its own package. - Create new package `util/net` and puts `LoopbackInterfaceName` there - Modify distributor_test to use `util/net` package * By default, append loopback to ingester interfaces. - Append the loopback net interface to the list of net interfaces used by the ingester ring. The append only occurs if the ingester net interfaces are default - Add CHANGELOG entry * Test ring net interface behavior. - Add tests to check the current net interface name resolution. The tests checks if the ingester ring net interface names are correctly copied to other rings
grafana · Oct 29, 2021 · 3c16658 · 3c16658
1 parent 96e6475
commit 3c16658
Show file tree

Hide file tree

Showing 5 changed files with 122 additions and 22 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,9 +9,10 @@
 * [4519](https://github.com/grafana/loki/pull/4519) **DylanGuedes** and **replay**: Loki: Enable FIFO cache by default
 * [4520](https://github.com/grafana/loki/pull/4520) **jordanrushing** and **owen-d**: Introduce overrides-exporter module for tenant limits
 * [4453](https://github.com/grafana/loki/pull/4453) **liguozhong**: Loki: Implement retry to s3 chunk storage
-* [4542](https://github.com/grafana/loki/pull/4542) **owen-d**: Introduce the `loki_overrides_defaults` metric and only export diffs for tenant limits.
+* [4542](https://github.com/grafana/loki/pull/4542) **owen-d**: Introduce the `loki_overrides_defaults` metric and only export diffs for tenant limits
 * [4498](https://github.com/grafana/loki/pull/4498) **trevorwhitney**: Feature: add virtual read and write targets
 * [4543](https://github.com/grafana/loki/pull/4543) **trevorwhitney**: Change more default values and improve application of common storage config
+* [4570](https://github.com/grafana/loki/pull/4570) **DylanGuedes**: Loki: Append loopback to ingester net interface default list
 
 # 2.3.0 (2021/08/06)
 

diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go
@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"math"
 	"math/rand"
-	"net"
 	"net/http"
 	"strconv"
 	"strings"
@@ -33,6 +32,7 @@ import (
 	"github.com/grafana/loki/pkg/logproto"
 	"github.com/grafana/loki/pkg/runtime"
 	fe "github.com/grafana/loki/pkg/util/flagext"
+	loki_net "github.com/grafana/loki/pkg/util/net"
 	"github.com/grafana/loki/pkg/validation"
 )
 
@@ -308,23 +308,6 @@ func TestDistributor_PushIngestionRateLimiter(t *testing.T) {
 	}
 }
 
-// loopbackInterfaceName search for the name of a loopback interface in the list
-// of the system's network interfaces.
-func loopbackInterfaceName() (string, error) {
-	is, err := net.Interfaces()
-	if err != nil {
-		return "", fmt.Errorf("can't retrieve loopback interface name: %s", err)
-	}
-
-	for _, i := range is {
-		if i.Flags&net.FlagLoopback != 0 {
-			return i.Name, nil
-		}
-	}
-
-	return "", fmt.Errorf("can't retrieve loopback interface name")
-}
-
 func prepare(t *testing.T, limits *validation.Limits, kvStore kv.Client, factory func(addr string) (ring_client.PoolClient, error)) *Distributor {
 	var (
 		distributorConfig Config
@@ -350,7 +333,7 @@ func prepare(t *testing.T, limits *validation.Limits, kvStore kv.Client, factory
 		})
 	}
 
-	loopbackName, err := loopbackInterfaceName()
+	loopbackName, err := loki_net.LoopbackInterfaceName()
 	require.NoError(t, err)
 
 	distributorConfig.DistributorRing.HeartbeatPeriod = 100 * time.Millisecond

diff --git a/pkg/loki/config_wrapper.go b/pkg/loki/config_wrapper.go
@@ -15,6 +15,7 @@ import (
 
 	loki_storage "github.com/grafana/loki/pkg/storage"
 	chunk_storage "github.com/grafana/loki/pkg/storage/chunk/storage"
+	loki_net "github.com/grafana/loki/pkg/util/net"
 )
 
 // ConfigWrapper is a struct containing the Loki config along with other values that can be set on the command line
@@ -75,8 +76,8 @@ func (c *ConfigWrapper) ApplyDynamicConfig() cfg.Source {
 			r.QueryScheduler.UseSchedulerRing = true
 		}
 
-		applyIngesterRingConfig(r)
 		applyPathPrefixDefaults(r, defaults)
+		applyIngesterRingConfig(r, &defaults)
 		applyMemberlistConfig(r)
 
 		if err := applyStorageConfig(r, &defaults); err != nil {
@@ -97,7 +98,14 @@ func (c *ConfigWrapper) ApplyDynamicConfig() cfg.Source {
 // we have a ring configured. The reason for centralizing on the ingester ring as this is been set in basically
 // all of our provided config files for all of time, usually set to `inmemory` for all the single binary Loki's
 // and is the most central ring config for Loki.
-func applyIngesterRingConfig(r *ConfigWrapper) {
+//
+// If the ingester ring has its interface names sets to a value equal to the default (["eth0", en0"]), it will try to append
+// the loopback interface at the end of it.
+func applyIngesterRingConfig(r *ConfigWrapper, defaults *ConfigWrapper) {
+	if reflect.DeepEqual(r.Ingester.LifecyclerConfig.InfNames, defaults.Ingester.LifecyclerConfig.InfNames) {
+		appendLoopbackInterface(r)
+	}
+
 	lc := r.Ingester.LifecyclerConfig
 	rc := r.Ingester.LifecyclerConfig.RingConfig
 	s := rc.KVStore.Store
@@ -158,6 +166,12 @@ func applyPathPrefixDefaults(r *ConfigWrapper, defaults ConfigWrapper) {
 	}
 }
 
+func appendLoopbackInterface(r *ConfigWrapper) {
+	if loopbackIface, err := loki_net.LoopbackInterfaceName(); err == nil {
+		r.Ingester.LifecyclerConfig.InfNames = append(r.Ingester.LifecyclerConfig.InfNames, loopbackIface)
+	}
+}
+
 // applyMemberlistConfig will change the default ingester, distributor, ruler, and query scheduler ring configurations to use memberlist
 // if the -memberlist.join_members config is provided. The idea here is that if a user explicitly configured the
 // memberlist configuration section, they probably want to be using memberlist for all their ring configurations.

diff --git a/pkg/loki/config_wrapper_test.go b/pkg/loki/config_wrapper_test.go
@@ -22,6 +22,7 @@ import (
 
 	"github.com/grafana/loki/pkg/storage/chunk/storage"
 	"github.com/grafana/loki/pkg/util/cfg"
+	loki_net "github.com/grafana/loki/pkg/util/net"
 )
 
 func configWrapperFromYAML(t *testing.T, configFileString string, args []string) (ConfigWrapper, ConfigWrapper, error) {
@@ -914,3 +915,81 @@ func Test_applyIngesterRingConfig(t *testing.T) {
 	assert.Equal(t, 8, reflect.TypeOf(distributor.RingConfig{}).NumField(), fmt.Sprintf(msgf, reflect.TypeOf(distributor.RingConfig{}).String()))
 
 }
+
+func TestRingInterfaceNames(t *testing.T) {
+	defaultIface, err := loki_net.LoopbackInterfaceName()
+	assert.NoError(t, err)
+	assert.NotEmpty(t, defaultIface)
+
+	t.Run("by default, loopback is available for all ring interfaces", func(t *testing.T) {
+		config, _, err := configWrapperFromYAML(t, minimalConfig, []string{})
+		assert.NoError(t, err)
+
+		assert.Contains(t, config.Ingester.LifecyclerConfig.InfNames, defaultIface)
+		assert.Contains(t, config.Distributor.DistributorRing.InstanceInterfaceNames, defaultIface)
+		assert.Contains(t, config.QueryScheduler.SchedulerRing.InstanceInterfaceNames, defaultIface)
+		assert.Contains(t, config.Ruler.Ring.InstanceInterfaceNames, defaultIface)
+	})
+
+	t.Run("if ingestor interface is set, it overrides other rings default interfaces", func(t *testing.T) {
+		yamlContent := `ingester:
+  lifecycler:
+    interface_names:
+    - ingesteriface`
+
+		config, _, err := configWrapperFromYAML(t, yamlContent, []string{})
+		assert.NoError(t, err)
+		assert.Equal(t, config.Distributor.DistributorRing.InstanceInterfaceNames, []string{"ingesteriface"})
+		assert.Equal(t, config.QueryScheduler.SchedulerRing.InstanceInterfaceNames, []string{"ingesteriface"})
+		assert.Equal(t, config.Ruler.Ring.InstanceInterfaceNames, []string{"ingesteriface"})
+		assert.Equal(t, config.Ingester.LifecyclerConfig.InfNames, []string{"ingesteriface"})
+	})
+
+	t.Run("if all rings have different net interface sets, doesn't override any of them", func(t *testing.T) {
+		yamlContent := `distributor:
+  ring:
+    instance_interface_names:
+    - distributoriface
+ruler:
+  ring:
+    instance_interface_names:
+    - ruleriface
+query_scheduler:
+  scheduler_ring:
+    instance_interface_names:
+    - scheduleriface
+ingester:
+  lifecycler:
+    interface_names:
+    - ingesteriface`
+
+		config, _, err := configWrapperFromYAML(t, yamlContent, []string{})
+		assert.NoError(t, err)
+		assert.Equal(t, config.Ingester.LifecyclerConfig.InfNames, []string{"ingesteriface"})
+		assert.Equal(t, config.Distributor.DistributorRing.InstanceInterfaceNames, []string{"distributoriface"})
+		assert.Equal(t, config.QueryScheduler.SchedulerRing.InstanceInterfaceNames, []string{"scheduleriface"})
+		assert.Equal(t, config.Ruler.Ring.InstanceInterfaceNames, []string{"ruleriface"})
+	})
+
+	t.Run("if all rings except ingester have net interface sets, doesn't override them with ingester default value", func(t *testing.T) {
+		yamlContent := `distributor:
+  ring:
+    instance_interface_names:
+    - distributoriface
+ruler:
+  ring:
+    instance_interface_names:
+    - ruleriface
+query_scheduler:
+  scheduler_ring:
+    instance_interface_names:
+    - scheduleriface`
+
+		config, _, err := configWrapperFromYAML(t, yamlContent, []string{})
+		assert.NoError(t, err)
+		assert.Equal(t, config.Distributor.DistributorRing.InstanceInterfaceNames, []string{"distributoriface"})
+		assert.Equal(t, config.QueryScheduler.SchedulerRing.InstanceInterfaceNames, []string{"scheduleriface"})
+		assert.Equal(t, config.Ruler.Ring.InstanceInterfaceNames, []string{"ruleriface"})
+		assert.Equal(t, config.Ingester.LifecyclerConfig.InfNames, []string{"eth0", "en0", defaultIface})
+	})
+}
diff --git a/pkg/util/net/net_interfaces.go b/pkg/util/net/net_interfaces.go
@@ -0,0 +1,23 @@
+package net
+
+import (
+	"fmt"
+	"net"
+)
+
+// LoopbackInterfaceName search for the name of a loopback interface in the list
+// of the system's network interfaces and returns the first one found.
+func LoopbackInterfaceName() (string, error) {
+	is, err := net.Interfaces()
+	if err != nil {
+		return "", fmt.Errorf("can't retrieve loopback interface name: %s", err)
+	}
+
+	for _, i := range is {
+		if i.Flags&net.FlagLoopback != 0 {
+			return i.Name, nil
+		}
+	}
+
+	return "", fmt.Errorf("can't retrieve loopback interface name")
+}