enable setting upstreams dynamically via HTTP endpoint

Makefile

@@ -50,7 +50,7 @@ IMAGE = $(REGISTRY)/$(IMGNAME)
 MULTI_ARCH_IMG = $(IMAGE)-$(ARCH)
 
 # Set default base image dynamically for each arch
-BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.34
+BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.37
 
 ifeq ($(ARCH),arm)
 	QEMUARCH=arm

cmd/nginx/flags.go

@@ -131,6 +131,9 @@ func parseFlags() (bool, *controller.Configuration, error) {
 		publishStatusAddress = flags.String("publish-status-address", "",
 			`User customized address to be set in the status of ingress resources. The controller will set the
 		endpoint records on the ingress using this address.`)
+
+		dynamicConfigurationEnabled = flags.Bool("enable-dynamic-configuration", false,
+			`When enabled controller will try to avoid Nginx reloads as much as possible by using Lua. Disabled by default.`)
 	)
 
 	flag.Set("logtostderr", "true")
@@ -214,6 +217,7 @@ func parseFlags() (bool, *controller.Configuration, error) {
 		SortBackends:                *sortBackends,
 		UseNodeInternalIP:           *useNodeInternalIP,
 		SyncRateLimit:               *syncRateLimit,
+		DynamicConfigurationEnabled: *dynamicConfigurationEnabled,
 		ListenPorts: &ngx_config.ListenPorts{
 			Default:  *defServerPort,
 			Health:   *healthzPort,

images/nginx/build.sh

@@ -32,7 +32,7 @@ export OPENTRACING_CPP_VERSION=1.2.0
 export ZIPKIN_CPP_VERSION=0.2.0
 export JAEGER_VERSION=0.2.0
 export MODSECURITY_VERSION=1.0.0
-export LUA_VERSION=0.10.12rc2
+export LUA_NGX_VERSION=0.10.12rc2
 export LUA_UPSTREAM_VERSION=0.07
 export COOKIE_FLAG_VERSION=1.1.0
 
@@ -155,7 +155,7 @@ get_src 9915ad1cf0734cc5b357b0d9ea92fec94764b4bf22f4dce185cbd65feda30ec1 \
         "https://github.com/AirisX/nginx_cookie_flag_module/archive/v$COOKIE_FLAG_VERSION.tar.gz"
 
 get_src 18edf2d18fa331265c36516a4a19ba75d26f46eafcc5e0c2d9aa6c237e8bc110 \
-        "https://github.com/openresty/lua-nginx-module/archive/v$LUA_VERSION.tar.gz"
+        "https://github.com/openresty/lua-nginx-module/archive/v$LUA_NGX_VERSION.tar.gz"
 
 get_src 2a69815e4ae01aa8b170941a8e1a10b6f6a9aab699dee485d58f021dd933829a \
         "https://github.com/openresty/lua-upstream-nginx-module/archive/v$LUA_UPSTREAM_VERSION.tar.gz"
@@ -166,6 +166,9 @@ get_src d4a9ed0d2405f41eb0178462b398afde8599c5115dcc1ff8f60e2f34a41a4c21 \
 get_src 92fd006d5ca3b3266847d33410eb280122a7f6c06334715f87acce064188a02e \
         "https://github.com/openresty/lua-resty-core/archive/v0.1.14rc1.tar.gz"
 
+get_src eaf84f58b43289c1c3e0442ada9ed40406357f203adc96e2091638080cb8d361 \
+        "https://github.com/openresty/lua-resty-lock/archive/v0.07.tar.gz"
+
 get_src 1ad2e34b111c802f9d0cdf019e986909123237a28c746b21295b63c9e785d9c3 \
         "http://luajit.org/download/LuaJIT-2.1.0-beta3.tar.gz"
 
@@ -197,6 +200,9 @@ make install
 cd "$BUILD_PATH/lua-resty-lrucache-0.07"
 make install
 
+cd "$BUILD_PATH/lua-resty-lock-0.07"
+make install
+
 # build opentracing lib
 cd "$BUILD_PATH/opentracing-cpp-$OPENTRACING_CPP_VERSION"
 mkdir .build
@@ -335,7 +341,7 @@ WITH_MODULES="--add-module=$BUILD_PATH/ngx_devel_kit-$NDK_VERSION \
   --add-module=$BUILD_PATH/nginx-goodies-nginx-sticky-module-ng-$STICKY_SESSIONS_VERSION \
   --add-module=$BUILD_PATH/nginx-http-auth-digest-$NGINX_DIGEST_AUTH \
   --add-module=$BUILD_PATH/ngx_http_substitutions_filter_module-$NGINX_SUBSTITUTIONS \
-  --add-module=$BUILD_PATH/lua-nginx-module-$LUA_VERSION \
+  --add-module=$BUILD_PATH/lua-nginx-module-$LUA_NGX_VERSION \
   --add-module=$BUILD_PATH/lua-upstream-nginx-module-$LUA_UPSTREAM_VERSION \
   --add-module=$BUILD_PATH/nginx_cookie_flag_module-$COOKIE_FLAG_VERSION \
   --add-dynamic-module=$BUILD_PATH/nginx-opentracing-$NGINX_OPENTRACING_VERSION/opentracing \

internal/ingress/controller/config/config.go

@@ -616,6 +616,7 @@ type TemplateConfig struct {
 	RedirectServers             map[string]string
 	ListenPorts                 *ListenPorts
 	PublishService              *apiv1.Service
+	DynamicConfigurationEnabled bool
 }
 
 // ListenPorts describe the ports required to run the

internal/ingress/controller/controller.go

@@ -95,6 +95,8 @@ type Configuration struct {
 	FakeCertificateSHA  string
 
 	SyncRateLimit float32
+
+	DynamicConfigurationEnabled bool
 }
 
 // GetPublishService returns the configured service used to set ingress status
@@ -167,11 +169,21 @@ func (n *NGINXController) syncIngress(item interface{}) error {
 	if !n.isForceReload() && n.runningConfig.Equal(&pcfg) {
 		glog.V(3).Infof("skipping backend reload (no changes detected)")
 		return nil
+	} else if !n.isForceReload() && n.cfg.DynamicConfigurationEnabled && n.IsDynamicallyConfigurable(&pcfg) {
+		err := n.ConfigureDynamically(&pcfg)
+		if err == nil {
+			glog.Infof("dynamic reconfiguration succeeded, skipping reload")
+			n.OnUpdate(pcfg, true)
+			n.runningConfig = &pcfg
+			return nil
+		}
+
+		glog.Warningf("falling back to reload, could not dynamically reconfigure: %v", err)
 	}
 
 	glog.Infof("backend reload required")
 
-	err := n.OnUpdate(pcfg)
+	err := n.OnUpdate(pcfg, false)
 	if err != nil {
 		incReloadErrorCount()
 		glog.Errorf("unexpected failure restarting the backend: \n%v", err)
@@ -182,6 +194,19 @@ func (n *NGINXController) syncIngress(item interface{}) error {
 	incReloadCount()
 	setSSLExpireTime(servers)
 
+	if n.isForceReload() && n.cfg.DynamicConfigurationEnabled {
+		go func() {
+			// it takes time for Nginx to start listening on the port
+			time.Sleep(1 * time.Second)
+			err := n.ConfigureDynamically(&pcfg)
+			if err == nil {
+				glog.Infof("dynamic reconfiguration succeeded")
+			} else {
+				glog.Warningf("could not dynamically reconfigure: %v", err)
+			}
+		}()
+	}
+
 	n.runningConfig = &pcfg
 	n.SetForceReload(false)
 

internal/ingress/controller/nginx.go

@@ -18,10 +18,12 @@ package controller
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
 	"net"
+	"net/http"
 	"os"
 	"os/exec"
 	"strconv"
@@ -448,7 +450,7 @@ Error: %v
 //
 // returning nill implies the backend will be reloaded.
 // if an error is returned means requeue the update
-func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error {
+func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration, skipReload bool) error {
 	cfg := n.store.GetBackendConfiguration()
 	cfg.Resolver = n.resolver
 
@@ -619,6 +621,7 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error {
 		IsSSLPassthroughEnabled:     n.cfg.EnableSSLPassthrough,
 		ListenPorts:                 n.cfg.ListenPorts,
 		PublishService:              n.GetPublishService(),
+		DynamicConfigurationEnabled: n.cfg.DynamicConfigurationEnabled,
 	}
 
 	content, err := n.t.Write(tc)
@@ -664,10 +667,12 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error {
 		return err
 	}
 
+	if !skipReload {
 		o, err := exec.Command(n.binary, "-s", "reload", "-c", cfgPath).CombinedOutput()
 		if err != nil {
 			return fmt.Errorf("%v\n%v", err, string(o))
 		}
+	}
 
 	return nil
 }
@@ -741,3 +746,54 @@ func (n *NGINXController) setupSSLProxy() {
 		}
 	}()
 }
+
+// IsDynamicallyConfigurable decides if the new configuration can be dynamically configured without reloading
+func (n *NGINXController) IsDynamicallyConfigurable(pcfg *ingress.Configuration) bool {
+	runningBackends := make([]*ingress.Backend, 0, len(n.runningConfig.Backends))
+	newBackends := make([]*ingress.Backend, 0, len(pcfg.Backends))
+
+	for i := 0; i < len(n.runningConfig.Backends); i++ {
+		runningBackends = append(runningBackends, n.runningConfig.Backends[i].DeepCopy())
+	}
+
+	for i := 0; i < len(pcfg.Backends); i++ {
+		newBackends = append(newBackends, pcfg.Backends[i].DeepCopy())
+	}
+
+	n.runningConfig.Backends = []*ingress.Backend{}
+	pcfg.Backends = []*ingress.Backend{}
+
+	ret := n.runningConfig.Equal(pcfg)
+
+	n.runningConfig.Backends = runningBackends
+	pcfg.Backends = newBackends
+
+	return ret
+}
+
+// ConfigureDynamically JSON encodes new Backends and POSTs it to an internal HTTP endpoint
+// that is handled by Lua
+func (n *NGINXController) ConfigureDynamically(pcfg *ingress.Configuration) error {
+	buf, err := json.Marshal(pcfg.Backends)
+	if err != nil {
+		return err
+	}
+
+	url := fmt.Sprintf("http://localhost:%d/configuration/backends", n.cfg.ListenPorts.Status)
+	resp, err := http.Post(url, "application/json", bytes.NewReader(buf))
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err := resp.Body.Close(); err != nil {
+			glog.Warningf("error while closing response body: \n%v", err)
+		}
+	}()
+
+	if resp.StatusCode != http.StatusCreated {
+		return fmt.Errorf("Unexpected error code: %d", resp.StatusCode)
+	}
+
+	return nil
+}

internal/ingress/controller/nginx_test.go

@@ -16,7 +16,67 @@ limitations under the License.
 
 package controller
 
-import "testing"
+import (
+	"testing"
+
+	"k8s.io/ingress-nginx/internal/ingress"
+)
+
+func TestIsDynamicallyConfigurable(t *testing.T) {
+	backends := []*ingress.Backend{{
+		Name: "fakenamespace-myapp-80",
+		Endpoints: []ingress.Endpoint{
+			{
+				Address: "10.0.0.1",
+				Port:    "8080",
+			},
+			{
+				Address: "10.0.0.2",
+				Port:    "8080",
+			},
+		},
+	}}
+
+	servers := []*ingress.Server{{
+		Hostname: "myapp.fake",
+		Locations: []*ingress.Location{
+			{
+				Path:    "/",
+				Backend: "fakenamespace-myapp-80",
+			},
+		},
+	}}
+
+	commonConfig := &ingress.Configuration{
+		Backends: backends,
+		Servers:  servers,
+	}
+
+	n := &NGINXController{
+		runningConfig: commonConfig,
+	}
+
+	newConfig := commonConfig
+	if !n.IsDynamicallyConfigurable(newConfig) {
+		t.Errorf("When new config is same as the running config it should be deemed as dynamically configurable")
+	}
+
+	newConfig = &ingress.Configuration{
+		Backends: []*ingress.Backend{{}},
+		Servers:  []*ingress.Server{{}},
+	}
+	if n.IsDynamicallyConfigurable(newConfig) {
+		t.Errorf("Expected to not be dynamically configurable when there's more than just backends change")
+	}
+
+	newConfig = &ingress.Configuration{
+		Backends: []*ingress.Backend{{}},
+		Servers:  servers,
+	}
+	if !n.IsDynamicallyConfigurable(newConfig) {
+		t.Errorf("Expected to be dynamically configurable when only backends change")
+	}
+}
 
 func TestNginxHashBucketSize(t *testing.T) {
 	tests := []struct {

internal/ingress/controller/template/template.go

@@ -281,7 +281,7 @@ func buildLogFormatUpstream(input interface{}) string {
 // (specified through the nginx.ingress.kubernetes.io/rewrite-to annotation)
 // If the annotation nginx.ingress.kubernetes.io/add-base-url:"true" is specified it will
 // add a base tag in the head of the response from the service
-func buildProxyPass(host string, b interface{}, loc interface{}) string {
+func buildProxyPass(host string, b interface{}, loc interface{}, dynamicConfigurationEnabled bool) string {
 	backends, ok := b.([]*ingress.Backend)
 	if !ok {
 		glog.Errorf("expected an '[]*ingress.Backend' type but %T was returned", b)
@@ -297,7 +297,10 @@ func buildProxyPass(host string, b interface{}, loc interface{}) string {
 	path := location.Path
 	proto := "http"
 
-	upstreamName := location.Backend
+	upstreamName := "upstream_balancer"
+
+	if !dynamicConfigurationEnabled {
+		upstreamName = location.Backend
 		for _, backend := range backends {
 			if backend.Name == location.Backend {
 				if backend.Secure || backend.SSLPassthrough {
@@ -311,9 +314,11 @@ func buildProxyPass(host string, b interface{}, loc interface{}) string {
 				break
 			}
 		}
+	}
 
 	// defProxyPass returns the default proxy_pass, just the name of the upstream
 	defProxyPass := fmt.Sprintf("proxy_pass %s://%s;", proto, upstreamName)
+
 	// if the path in the ingress rule is equals to the target: no special rewrite
 	if path == location.Rewrite.Target {
 		return defProxyPass