Add ability for nginx service mesh to egress through a virtualserver …

…resource - added internalRoute field to the virtualserver CRD - added templates for internal routes in virtualserver templates for n+ and oss - added unit test to validate virtualserver internal routes - added enableInternalRoutes boolean to virtualServerConfigurator type - updated virtualserver configuration items to include internRoute docs
nginxinc · Feb 27, 2023 · 7d1ca30 · 7d1ca30
1 parent 21f5377
commit 7d1ca30
Show file tree

Hide file tree

Showing 9 changed files with 161 additions and 14 deletions.
diff --git a/deployments/common/crds/k8s.nginx.org_virtualservers.yaml b/deployments/common/crds/k8s.nginx.org_virtualservers.yaml
@@ -93,6 +93,9 @@ spec:
                   type: string
                 ingressClassName:
                   type: string
+                internalRoute:
+                  description: internalRoute is used to toggle internal routing used for specific types of traffic egress.
+                  type: boolean
                 policies:
                   type: array
                   items:

diff --git a/deployments/helm-chart/crds/k8s.nginx.org_virtualservers.yaml b/deployments/helm-chart/crds/k8s.nginx.org_virtualservers.yaml
@@ -93,6 +93,9 @@ spec:
                   type: string
                 ingressClassName:
                   type: string
+                internalRoute:
+                  description: internalRoute is used to toggle internal routing used for specific types of traffic egress.
+                  type: boolean
                 policies:
                   type: array
                   items:

diff --git a/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md b/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md
@@ -59,6 +59,7 @@ spec:
 |``upstreams`` | A list of upstreams. | [[]upstream](#upstream) | No |
 |``routes`` | A list of routes. | [[]route](#virtualserverroute) | No |
 |``ingressClassName`` | Specifies which Ingress Controller must handle the VirtualServer resource. | ``string`` | No |
+|``internalRoute`` | Specifies if the VirtualServer resource is an internal route or not. | ``boolean`` | No |
 |``http-snippets`` | Sets a custom snippet in the http context. | ``string`` | No |
 |``server-snippets`` | Sets a custom snippet in server context. Overrides the ``server-snippets`` ConfigMap key. | ``string`` | No |
 {{% /table %}}

diff --git a/internal/configs/version2/http.go b/internal/configs/version2/http.go
@@ -12,14 +12,15 @@ type UpstreamLabels struct {
 
 // VirtualServerConfig holds NGINX configuration for a VirtualServer.
 type VirtualServerConfig struct {
-	HTTPSnippets  []string
-	LimitReqZones []LimitReqZone
-	Maps          []Map
-	Server        Server
-	SpiffeCerts   bool
-	SplitClients  []SplitClient
-	StatusMatches []StatusMatch
-	Upstreams     []Upstream
+	HTTPSnippets        []string
+	LimitReqZones       []LimitReqZone
+	Maps                []Map
+	Server              Server
+	SpiffeCerts         bool
+	InternalRouteServer bool
+	SplitClients        []SplitClient
+	StatusMatches       []StatusMatch
+	Upstreams           []Upstream
 }
 
 // Upstream defines an upstream.

diff --git a/internal/configs/version2/nginx-plus.virtualserver.tmpl b/internal/configs/version2/nginx-plus.virtualserver.tmpl
@@ -68,8 +68,17 @@ match {{ $m.Name }} {
 {{ end }}
 
 server {
+
+    {{ if $.InternalRouteServer }}
+    listen 443 ssl;
+    {{if not $s.DisableIPV6}}listen [::]:443 ssl;{{end}}
+    ssl_certificate /etc/nginx/secrets/spiffe_cert.pem;
+    ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem;
+
+    {{ else }}
     listen 80{{ if $s.ProxyProtocol }} proxy_protocol{{ end }};
     {{ if not $s.DisableIPV6 }}listen [::]:80{{ if $s.ProxyProtocol }} proxy_protocol{{ end }};{{ end }}
+    {{ end }}
 
     server_name {{ $s.ServerName }};
     status_zone {{ $s.StatusZone }};

diff --git a/internal/configs/version2/nginx.virtualserver.tmpl b/internal/configs/version2/nginx.virtualserver.tmpl
@@ -40,8 +40,17 @@ limit_req_zone {{ $z.Key }} zone={{ $z.ZoneName }}:{{ $z.ZoneSize }} rate={{ $z.
 
 {{ $s := .Server }}
 server {
+
+    {{ if $.InternalRouteServer }}
+    listen 443 ssl;
+    {{if not $s.DisableIPV6}}listen [::]:443 ssl;{{end}}
+    ssl_certificate /etc/nginx/secrets/spiffe_cert.pem;
+    ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem;
+
+    {{ else }}
     listen 80{{ if $s.ProxyProtocol }} proxy_protocol{{ end }};
     {{ if not $s.DisableIPV6 }}listen [::]:80{{ if $s.ProxyProtocol }} proxy_protocol{{ end }};{{ end }}
+    {{ end }}
 
     server_name {{ $s.ServerName }};
 
@@ -310,7 +319,6 @@ server {
             {{ if $l.ProxyBufferSize }}
         {{ $proxyOrGRPC }}_buffer_size {{ $l.ProxyBufferSize }};
             {{ end }}
-
             {{ if not $l.GRPCPass }}
         proxy_http_version 1.1;
         set $default_connection_header {{ if $l.HasKeepalive }}""{{ else }}close{{ end }};

diff --git a/internal/configs/virtualserver.go b/internal/configs/virtualserver.go
@@ -233,6 +233,7 @@ type virtualServerConfigurator struct {
 	enableSnippets       bool
 	warnings             Warnings
 	spiffeCerts          bool
+	enableInternalRoutes bool
 	oidcPolCfg           *oidcPolicyCfg
 	isIPV6Disabled       bool
 }
@@ -273,6 +274,7 @@ func newVirtualServerConfigurator(
 		enableSnippets:       staticParams.EnableSnippets,
 		warnings:             make(map[runtime.Object][]string),
 		spiffeCerts:          staticParams.NginxServiceMesh,
+		enableInternalRoutes: staticParams.EnableInternalRoutes,
 		oidcPolCfg:           &oidcPolicyCfg{},
 		isIPV6Disabled:       staticParams.DisableIPV6,
 	}
@@ -356,7 +358,7 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig(
 		ups := vsc.generateUpstream(vsEx.VirtualServer, upstreamName, u, isExternalNameSvc, endpoints)
 		upstreams = append(upstreams, ups)
 
-		u.TLS.Enable = isTLSEnabled(u, vsc.spiffeCerts)
+		u.TLS.Enable = isTLSEnabled(u, vsc.spiffeCerts, vsEx.VirtualServer.Spec.InternalRoute)
 		crUpstreams[upstreamName] = u
 
 		if hc := generateHealthCheck(u, upstreamName, vsc.cfgParams); hc != nil {
@@ -385,7 +387,7 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig(
 			_, isExternalNameSvc := vsEx.ExternalNameSvcs[GenerateExternalNameSvcKey(upstreamNamespace, u.Service)]
 			ups := vsc.generateUpstream(vsr, upstreamName, u, isExternalNameSvc, endpoints)
 			upstreams = append(upstreams, ups)
-			u.TLS.Enable = isTLSEnabled(u, vsc.spiffeCerts)
+			u.TLS.Enable = isTLSEnabled(u, vsc.spiffeCerts, vsEx.VirtualServer.Spec.InternalRoute)
 			crUpstreams[upstreamName] = u
 
 			if hc := generateHealthCheck(u, upstreamName, vsc.cfgParams); hc != nil {
@@ -677,7 +679,8 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig(
 			VSName:                    vsEx.VirtualServer.Name,
 			DisableIPV6:               vsc.isIPV6Disabled,
 		},
-		SpiffeCerts: vsc.spiffeCerts,
+		SpiffeCerts:         vsc.spiffeCerts,
+		InternalRouteServer: vsEx.VirtualServer.Spec.InternalRoute,
 	}
 
 	return vsCfg, vsc.warnings
@@ -2448,7 +2451,11 @@ func generateProxySSLName(svcName, ns string) string {
 	return fmt.Sprintf("%s.%s.svc", svcName, ns)
 }
 
-func isTLSEnabled(u conf_v1.Upstream, spiffeCerts bool) bool {
+func isTLSEnabled(u conf_v1.Upstream, spiffeCerts, spiffeEgress bool) bool {
+	if spiffeEgress {
+		return false
+	}
+
 	return u.TLS.Enable || spiffeCerts
 }
 

diff --git a/internal/configs/virtualserver_test.go b/internal/configs/virtualserver_test.go
@@ -1275,6 +1275,119 @@ func TestGenerateVirtualServerConfigWithSpiffeCerts(t *testing.T) {
 	}
 }
 
+func TestGenerateVirtualServerConfigWithInternalRoutes(t *testing.T) {
+	t.Parallel()
+	virtualServerEx := VirtualServerEx{
+		VirtualServer: &conf_v1.VirtualServer{
+			ObjectMeta: meta_v1.ObjectMeta{
+				Name:      "cafe",
+				Namespace: "default",
+			},
+			Spec: conf_v1.VirtualServerSpec{
+				Host: "cafe.example.com",
+				Upstreams: []conf_v1.Upstream{
+					{
+						Name:    "tea",
+						Service: "tea-svc",
+						Port:    80,
+						TLS:     conf_v1.UpstreamTLS{Enable: false},
+					},
+				},
+				Routes: []conf_v1.Route{
+					{
+						Path: "/",
+						Action: &conf_v1.Action{
+							Pass: "tea",
+						},
+					},
+				},
+				InternalRoute: true,
+			},
+		},
+		Endpoints: map[string][]string{
+			"default/tea-svc:80": {
+				"10.0.0.20:80",
+			},
+		},
+	}
+
+	baseCfgParams := ConfigParams{
+		ServerTokens:    "off",
+		Keepalive:       16,
+		ServerSnippets:  []string{"# server snippet"},
+		ProxyProtocol:   true,
+		SetRealIPFrom:   []string{"0.0.0.0/0"},
+		RealIPHeader:    "X-Real-IP",
+		RealIPRecursive: true,
+	}
+
+	expected := version2.VirtualServerConfig{
+		Upstreams: []version2.Upstream{
+			{
+				UpstreamLabels: version2.UpstreamLabels{
+					Service:           "tea-svc",
+					ResourceType:      "virtualserver",
+					ResourceName:      "cafe",
+					ResourceNamespace: "default",
+				},
+				Name: "vs_default_cafe_tea",
+				Servers: []version2.UpstreamServer{
+					{
+						Address: "10.0.0.20:80",
+					},
+				},
+				Keepalive: 16,
+			},
+		},
+		HTTPSnippets:  []string{},
+		LimitReqZones: []version2.LimitReqZone{},
+		Server: version2.Server{
+			ServerName:      "cafe.example.com",
+			StatusZone:      "cafe.example.com",
+			VSNamespace:     "default",
+			VSName:          "cafe",
+			ProxyProtocol:   true,
+			ServerTokens:    "off",
+			SetRealIPFrom:   []string{"0.0.0.0/0"},
+			RealIPHeader:    "X-Real-IP",
+			RealIPRecursive: true,
+			Snippets:        []string{"# server snippet"},
+			TLSPassthrough:  true,
+			Locations: []version2.Location{
+				{
+					Path:                     "/",
+					ProxyPass:                "http://vs_default_cafe_tea",
+					ProxyNextUpstream:        "error timeout",
+					ProxyNextUpstreamTimeout: "0s",
+					ProxyNextUpstreamTries:   0,
+					HasKeepalive:             true,
+					ProxySSLName:             "tea-svc.default.svc",
+					ProxyPassRequestHeaders:  true,
+					ProxySetHeaders:          []version2.Header{{Name: "Host", Value: "$host"}},
+					ServiceName:              "tea-svc",
+				},
+			},
+		},
+		SpiffeCerts:         true,
+		InternalRouteServer: true,
+	}
+
+	isPlus := false
+	isResolverConfigured := false
+	staticConfigParams := &StaticConfigParams{TLSPassthrough: true, NginxServiceMesh: true, EnableInternalRoutes: true}
+	isWildcardEnabled := false
+	vsc := newVirtualServerConfigurator(&baseCfgParams, isPlus, isResolverConfigured, staticConfigParams, isWildcardEnabled)
+
+	result, warnings := vsc.GenerateVirtualServerConfig(&virtualServerEx, nil, nil)
+	if diff := cmp.Diff(expected, result); diff != "" {
+		t.Errorf("GenerateVirtualServerConfig() mismatch (-want +got):\n%s", diff)
+	}
+
+	if len(warnings) != 0 {
+		t.Errorf("GenerateVirtualServerConfig returned warnings: %v", vsc.warnings)
+	}
+}
+
 func TestGenerateVirtualServerConfigForVirtualServerWithSplits(t *testing.T) {
 	t.Parallel()
 	virtualServerEx := VirtualServerEx{
@@ -8061,6 +8174,7 @@ func TestIsTLSEnabled(t *testing.T) {
 	tests := []struct {
 		upstream   conf_v1.Upstream
 		spiffeCert bool
+		nsmEgress  bool
 		expected   bool
 	}{
 		{
@@ -8102,7 +8216,7 @@ func TestIsTLSEnabled(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		result := isTLSEnabled(test.upstream, test.spiffeCert)
+		result := isTLSEnabled(test.upstream, test.spiffeCert, test.nsmEgress)
 		if result != test.expected {
 			t.Errorf("isTLSEnabled(%v, %v) returned %v but expected %v", test.upstream, test.spiffeCert, result, test.expected)
 		}

diff --git a/pkg/apis/configuration/v1/types.go b/pkg/apis/configuration/v1/types.go
@@ -46,6 +46,7 @@ type VirtualServerSpec struct {
 	ServerSnippets string            `json:"server-snippets"`
 	Dos            string            `json:"dos"`
 	ExternalDNS    ExternalDNS       `json:"externalDNS"`
+	InternalRoute  bool              `json:"internalRoute"`
 }
 
 // ExternalDNS defines externaldns sub-resource of a virtual server.