net 2898, graceful_startup

.changelog/239.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+Add graceful_startup endpoint and postStart hook in order to guarantee that dataplane starts up before application container. 
+```

cmd/consul-dataplane/config.go

@@ -121,6 +121,10 @@ type EnvoyFlags struct {
 	GracefulShutdownPath          *string `json:"gracefulShutdownPath,omitempty"`
 	GracefulPort                  *int    `json:"gracefulPort,omitempty"`
 	DumpEnvoyConfigOnExitEnabled  *bool   `json:"dumpEnvoyConfigOnExitEnabled,omitempty"`
+	//Time in seconds to wait for dataplane to be ready.
+	StartupGracePeriodSeconds *int `json:"startupGracePeriodSeconds,omitempty"`
+	//Endpoint for graceful startup function.
+	GracefulStartupPath *string `json:"gracefulStartupPath,omitempty"`
 }
 
 const (
@@ -217,7 +221,9 @@ func buildDefaultConsulDPFlags() (DataplaneConfigFlags, error) {
 			"shutdownGracePeriodSeconds": 0,
 			"gracefulShutdownPath": "/graceful_shutdown",
 			"gracefulPort": 20300,
-			"dumpEnvoyConfigOnExitEnabled": false
+			"dumpEnvoyConfigOnExitEnabled": false,
+			"gracefulStartupPath": "/graceful_startup",
+			"startupGracePeriodSeconds": 0
 		},
 		"xdsServer": {
 			"bindAddress": "127.0.0.1",
@@ -295,6 +301,8 @@ func constructRuntimeConfig(cfg DataplaneConfigFlags, extraArgs []string) (*cons
 			DumpEnvoyConfigOnExitEnabled:  boolVal(cfg.Envoy.DumpEnvoyConfigOnExitEnabled),
 			GracefulShutdownPath:          stringVal(cfg.Envoy.GracefulShutdownPath),
 			GracefulPort:                  intVal(cfg.Envoy.GracefulPort),
+			StartupGracePeriodSeconds:     intVal(cfg.Envoy.StartupGracePeriodSeconds),
+			GracefulStartupPath:           stringVal(cfg.Envoy.GracefulStartupPath),
 			ExtraArgs:                     extraArgs,
 		},
 		Telemetry: &consuldp.TelemetryConfig{

cmd/consul-dataplane/config_test.go

@@ -86,6 +86,7 @@ func TestConfigGeneration(t *testing.T) {
 						GracefulShutdownPath:          "/graceful_shutdown",
 						EnvoyDrainTimeSeconds:         30,
 						GracefulPort:                  20300,
+						GracefulStartupPath:           "/graceful_startup",
 					},
 					Telemetry: &consuldp.TelemetryConfig{
 						UseCentralConfig: true,
@@ -189,6 +190,7 @@ func TestConfigGeneration(t *testing.T) {
 						EnvoyDrainTimeSeconds:         30,
 						GracefulPort:                  20300,
 						DumpEnvoyConfigOnExitEnabled:  true,
+						GracefulStartupPath:           "/graceful_startup",
 					},
 					Telemetry: &consuldp.TelemetryConfig{
 						UseCentralConfig: true,
@@ -287,6 +289,7 @@ func TestConfigGeneration(t *testing.T) {
 						EnvoyDrainTimeSeconds:         30,
 						GracefulPort:                  20300,
 						DumpEnvoyConfigOnExitEnabled:  false,
+						GracefulStartupPath:           "/graceful_startup",
 					},
 					Telemetry: &consuldp.TelemetryConfig{
 						UseCentralConfig: true,
@@ -410,6 +413,7 @@ func TestConfigGeneration(t *testing.T) {
 						EnvoyDrainTimeSeconds:         30,
 						GracefulPort:                  20300,
 						DumpEnvoyConfigOnExitEnabled:  false,
+						GracefulStartupPath:           "/graceful_startup",
 					},
 					Telemetry: &consuldp.TelemetryConfig{
 						UseCentralConfig: true,

cmd/consul-dataplane/main.go

@@ -99,7 +99,8 @@ func init() {
 	IntVar(flags, &flagOpts.dataplaneConfig.Envoy.ShutdownGracePeriodSeconds, "shutdown-grace-period-seconds", "DP_SHUTDOWN_GRACE_PERIOD_SECONDS", "Amount of time to wait after receiving a SIGTERM signal before terminating the proxy.")
 	StringVar(flags, &flagOpts.dataplaneConfig.Envoy.GracefulShutdownPath, "graceful-shutdown-path", "DP_GRACEFUL_SHUTDOWN_PATH", "An HTTP path to serve the graceful shutdown endpoint.")
 	IntVar(flags, &flagOpts.dataplaneConfig.Envoy.GracefulPort, "graceful-port", "DP_GRACEFUL_PORT", "A port to serve HTTP endpoints for graceful shutdown.")
-
+	IntVar(flags, &flagOpts.dataplaneConfig.Envoy.StartupGracePeriodSeconds, "startup-grace-period-seconds", "DP_STARTUP_GRACE_PERIOD_SECONDS", "Amount of time to wait for consul-dataplane startup.")
+	StringVar(flags, &flagOpts.dataplaneConfig.Envoy.GracefulStartupPath, "graceful-startup-path", "DP_GRACEFUL_STARTUP_PATH", "An HTTP path to serve the graceful startup endpoint.")
 	// Default is false, may be useful for debugging unexpected termination.
 	BoolVar(flags, &flagOpts.dataplaneConfig.Envoy.DumpEnvoyConfigOnExitEnabled, "dump-envoy-config-on-exit", "DP_DUMP_ENVOY_CONFIG_ON_EXIT", "Call the Envoy /config_dump endpoint during consul-dataplane controlled shutdown.")
 

pkg/consuldp/config.go

@@ -291,8 +291,12 @@ type EnvoyConfig struct {
 	ShutdownDrainListenersEnabled bool
 	// ShutdownGracePeriodSeconds is the amount of time to wait after receiving a SIGTERM before terminating the proxy container.
 	ShutdownGracePeriodSeconds int
-	// GracefulShutdownPath is the path on which the HTTP endpoint to initiate a graceful shutdown of Envoy is served
+	// GracefulShutdownPath is the path on which the HTTP endpoint to initiate a graceful shutdown of Envoy is served.
 	GracefulShutdownPath string
+	// StartupGracePeriodSeconds is the amount of time to block application after startup for Envoy proxy to be ready.
+	StartupGracePeriodSeconds int
+	// GracefulStartupPath is the path where the HTTP endpoint to initiate a graceful startup of Envoy is served.
+	GracefulStartupPath string
 	// GracefulPort is the port on which the HTTP server for graceful shutdown endpoints will be available.
 	GracefulPort int
 	// DumpEnvoyConfigOnExitEnabled configures whether to call Envoy's /config_dump endpoint during consul-dataplane controlled shutdown.

pkg/consuldp/lifecycle.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"strconv"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/hashicorp/go-hclog"
@@ -24,6 +25,7 @@ const (
 	cdpLifecycleUrl          = "http://" + cdpLifecycleBindAddr
 
 	defaultLifecycleShutdownPath = "/graceful_shutdown"
+	defaultLifecycleStartupPath  = "/graceful_startup"
 )
 
 // lifecycleConfig handles all configuration related to managing the Envoy proxy
@@ -36,8 +38,9 @@ type lifecycleConfig struct {
 	shutdownGracePeriodSeconds    int
 	gracefulPort                  int
 	gracefulShutdownPath          string
-
-	dumpEnvoyConfigOnExitEnabled bool
+	startupGracePeriodSeconds     int
+	gracefulStartupPath           string
+	dumpEnvoyConfigOnExitEnabled  bool
 
 	// manager for controlling the Envoy proxy process
 	proxy envoy.ProxyManager
@@ -58,8 +61,9 @@ func NewLifecycleConfig(cfg *Config, proxy envoy.ProxyManager) *lifecycleConfig
 		gracefulPort:                  cfg.Envoy.GracefulPort,
 		gracefulShutdownPath:          cfg.Envoy.GracefulShutdownPath,
 		dumpEnvoyConfigOnExitEnabled:  cfg.Envoy.DumpEnvoyConfigOnExitEnabled,
-
-		proxy: proxy,
+		startupGracePeriodSeconds:     cfg.Envoy.StartupGracePeriodSeconds,
+		gracefulStartupPath:           cfg.Envoy.GracefulStartupPath,
+		proxy:                         proxy,
 
 		errorExitCh: make(chan struct{}, 1),
 		mu:          sync.Mutex{},
@@ -84,18 +88,11 @@ func (m *lifecycleConfig) startLifecycleManager(ctx context.Context) error {
 	// management control
 	mux := http.NewServeMux()
 
-	// Determine what HTTP endpoint paths to configure for the proxy lifecycle
-	// management server. These can be set as flags.
-	cdpLifecycleShutdownPath := defaultLifecycleShutdownPath
-	if m.gracefulShutdownPath != "" {
-		cdpLifecycleShutdownPath = m.gracefulShutdownPath
-	}
-
-	// Set config to allow introspection of default path for testing
-	m.gracefulShutdownPath = cdpLifecycleShutdownPath
+	m.logger.Info(fmt.Sprintf("setting graceful shutdown path: %s\n", m.shutdownPath()))
+	mux.HandleFunc(m.shutdownPath(), m.gracefulShutdownHandler)
 
-	m.logger.Info(fmt.Sprintf("setting graceful shutdown path: %s\n", cdpLifecycleShutdownPath))
-	mux.HandleFunc(cdpLifecycleShutdownPath, m.gracefulShutdownHandler)
+	m.logger.Info(fmt.Sprintf("setting graceful startup path: %s\n", m.startupPath()))
+	mux.HandleFunc(m.startupPath(), m.gracefulStartupHandler)
 
 	// Determine what the proxy lifecycle management server bind port is. It can be
 	// set as a flag.
@@ -211,3 +208,60 @@ func (m *lifecycleConfig) gracefulShutdown() {
 	// Wait for context timeout to elapse
 	wg.Wait()
 }
+
+func (m *lifecycleConfig) gracefulStartupHandler(rw http.ResponseWriter, _ *http.Request) {
+	//Unlike in gracefulShutdown, we want to delay the OK response until envoy is ready
+	//in order to block application container.
+	m.gracefulStartup()
+	rw.WriteHeader(http.StatusOK)
+}
+
+// gracefulStartup blocks until the startup grace period has elapsed or we have confirmed that
+// Envoy proxy is ready.
+func (m *lifecycleConfig) gracefulStartup() {
+	if m.startupGracePeriodSeconds == 0 {
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(m.startupGracePeriodSeconds)*time.Second)
+	defer cancel()
+
+	var ready atomic.Bool
+	go func() {
+		for ctx.Err() == nil {
+			r, err := m.proxy.Ready()
+			if err != nil {
+				m.logger.Info(fmt.Sprintf("error when querying proxy readiness, %s", err.Error()))
+			}
+			if r {
+				ready.Store(true)
+				cancel()
+				break
+			}
+			time.Sleep(50 * time.Millisecond)
+		}
+	}()
+
+	<-ctx.Done()
+	if !ready.Load() {
+		m.logger.Warn("grace period elapsed before proxy ready")
+	}
+}
+
+func (m *lifecycleConfig) shutdownPath() string {
+	if m.gracefulShutdownPath == "" {
+		// Set config to allow introspection of default path for testing
+		m.gracefulShutdownPath = defaultLifecycleShutdownPath
+	}
+
+	return m.gracefulShutdownPath
+}
+
+func (m *lifecycleConfig) startupPath() string {
+	if m.gracefulStartupPath == "" {
+		// Set config to allow introspection of default path for testing
+		m.gracefulStartupPath = defaultLifecycleStartupPath
+	}
+
+	return m.gracefulStartupPath
+}