Issue #240: Retry registry during startup

This patch makes fabio retry connecting to the configured
backend registry repeatedly until it succeeds or a timeout
occurrs.

Fixes #240

config/config.go

@@ -87,6 +87,8 @@ type Registry struct {
 	Static  Static
 	File    File
 	Consul  Consul
+	Timeout time.Duration
+	Retry   time.Duration
 }
 
 type Static struct {

config/default.go

@@ -41,6 +41,8 @@ var defaultConfig = &Config{
 			CheckInterval: time.Second,
 			CheckTimeout:  3 * time.Second,
 		},
+		Timeout: 10 * time.Second,
+		Retry:   500 * time.Millisecond,
 	},
 	Runtime: Runtime{
 		GOGC:       800,

config/load.go

@@ -144,6 +144,8 @@ func load(cmdline, environ, envprefix []string, props *properties.Properties) (c
 	f.StringVar(&cfg.Metrics.Circonus.BrokerID, "metrics.circonus.brokerid", defaultConfig.Metrics.Circonus.BrokerID, "Circonus Broker ID")
 	f.StringVar(&cfg.Metrics.Circonus.CheckID, "metrics.circonus.checkid", defaultConfig.Metrics.Circonus.CheckID, "Circonus Check ID")
 	f.StringVar(&cfg.Registry.Backend, "registry.backend", defaultConfig.Registry.Backend, "registry backend")
+	f.DurationVar(&cfg.Registry.Timeout, "registry.timeout", defaultConfig.Registry.Timeout, "timeout for registry to become available")
+	f.DurationVar(&cfg.Registry.Retry, "registry.retry", defaultConfig.Registry.Retry, "retry interval during startup")
 	f.StringVar(&cfg.Registry.File.Path, "registry.file.path", defaultConfig.Registry.File.Path, "path to file based routing table")
 	f.StringVar(&cfg.Registry.Static.Routes, "registry.static.routes", defaultConfig.Registry.Static.Routes, "static routes")
 	f.StringVar(&cfg.Registry.Consul.Addr, "registry.consul.addr", defaultConfig.Registry.Consul.Addr, "address of the consul agent")

config/load_test.go

@@ -302,6 +302,20 @@ func TestLoad(t *testing.T) {
 				return cfg
 			},
 		},
+		{
+			args: []string{"-registry.timeout", "5s"},
+			cfg: func(cfg *Config) *Config {
+				cfg.Registry.Timeout = 5 * time.Second
+				return cfg
+			},
+		},
+		{
+			args: []string{"-registry.retry", "500ms"},
+			cfg: func(cfg *Config) *Config {
+				cfg.Registry.Retry = 500 * time.Millisecond
+				return cfg
+			},
+		},
 		{
 			args: []string{"-registry.file.path", "value"},
 			cfg: func(cfg *Config) *Config {

fabio.properties

@@ -392,6 +392,22 @@
 # registry.backend = consul
 
 
+# registry.timeout configures how long fabio tries to connect to the registry
+# backend during startup.
+#
+# The default is
+#
+# registry.timeout = 10s
+
+
+# registry.retry configures the interval with which fabio tries to
+# connect to the registry during startup.
+#
+# The default is
+#
+# registry.retry = 500ms
+
+
 # registry.static.routes configures a static routing table.
 #
 # Example:

main.go

@@ -11,6 +11,7 @@ import (
 	"runtime"
 	"runtime/debug"
 	"strings"
+	"time"
 
 	"github.com/eBay/fabio/admin"
 	"github.com/eBay/fabio/config"
@@ -143,24 +144,33 @@ func initRuntime(cfg *config.Config) {
 }
 
 func initBackend(cfg *config.Config) {
+	var deadline = time.Now().Add(cfg.Registry.Timeout)
+
 	var err error
+	for {
+		switch cfg.Registry.Backend {
+		case "file":
+			registry.Default, err = file.NewBackend(cfg.Registry.File.Path)
+		case "static":
+			registry.Default, err = static.NewBackend(cfg.Registry.Static.Routes)
+		case "consul":
+			registry.Default, err = consul.NewBackend(&cfg.Registry.Consul)
+		default:
+			exit.Fatal("[FATAL] Unknown registry backend ", cfg.Registry.Backend)
+		}
 
-	switch cfg.Registry.Backend {
-	case "file":
-		registry.Default, err = file.NewBackend(cfg.Registry.File.Path)
-	case "static":
-		registry.Default, err = static.NewBackend(cfg.Registry.Static.Routes)
-	case "consul":
-		registry.Default, err = consul.NewBackend(&cfg.Registry.Consul)
-	default:
-		exit.Fatal("[FATAL] Unknown registry backend ", cfg.Registry.Backend)
-	}
+		if err == nil {
+			if err = registry.Default.Register(); err == nil {
+				return
+			}
+		}
+		log.Print("[WARN] Error initializing backend. ", err)
 
-	if err != nil {
-		exit.Fatal("[FATAL] Error initializing backend. ", err)
-	}
-	if err := registry.Default.Register(); err != nil {
-		exit.Fatal("[FATAL] Error registering backend. ", err)
+		if time.Now().After(deadline) {
+			exit.Fatal("[FATAL] Timeout registering backend.")
+		}
+
+		time.Sleep(cfg.Registry.Retry)
 	}
 }