Mtls verify annotation support in native ingress controller

pkg/controllers/ingress/ingress.go

@@ -41,6 +41,7 @@ import (
 	"k8s.io/client-go/util/retry"
 	"k8s.io/client-go/util/workqueue"
 
+	"github.com/oracle/oci-go-sdk/v65/common"
 	ociloadbalancer "github.com/oracle/oci-go-sdk/v65/loadbalancer"
 )
 
@@ -366,11 +367,41 @@ func (c *Controller) ensureIngress(ingress *networkingv1.Ingress, ingressClass *
 
 		var listenerSslConfig *ociloadbalancer.SslConfigurationDetails
 		artifact, artifactType := stateStore.GetTLSConfigForListener(port)
+
 		listenerSslConfig, err := GetSSLConfigForListener(ingress.Namespace, nil, artifactType, artifact, c.defaultCompartmentId, c.client)
 		if err != nil {
 			return err
 		}
 
+		// listenerSslConfig.VerifyPeerCertificate
+
+		mode, deepth, trustcacert := stateStore.GetMutualTlsPortConfigForListener(port)
+		mtlsPorts := stateStore.IngressGroupState.MtlsPorts
+		klog.Infof("  GetMutualTlsPortConfigForListener **********   mtlsPorts :  %s ", util.PrettyPrint(mtlsPorts))
+
+		klog.Infof("  GetMutualTlsPortConfigForListener **********  before :  %s ", util.PrettyPrint(listenerSslConfig))
+		if mode == util.MutualTlsAuthenticationVerify {
+			// listenerSslConfig.VerifyDepth
+			listenerSslConfig.VerifyPeerCertificate = common.Bool(true)
+			listenerSslConfig.VerifyDepth = &deepth
+
+			// check wethear the trustcacert is valid ca bundle
+			if mode == util.MutualTlsAuthenticationVerify && isTrustAuthorityCaBundle(trustcacert) {
+				caBundleIds := []string{trustcacert}
+				listenerSslConfig.TrustedCertificateAuthorityIds = caBundleIds
+			} else {
+				if mode == util.MutualTlsAuthenticationVerify {
+					klog.Error("verify trustcacert error, not a valid CA bundle ID: %s", util.PrettyPrint(trustcacert))
+				}
+				listenerSslConfig.VerifyPeerCertificate = common.Bool(false)
+				listenerSslConfig.VerifyDepth = common.Int(1)
+				listenerSslConfig.TrustedCertificateAuthorityIds = nil
+
+			}
+
+		}
+		klog.Infof("  GetMutualTlsPortConfigForListener  after :  %s ", util.PrettyPrint(listenerSslConfig))
+
 		protocol := stateStore.GetListenerProtocol(port)
 		defaultBackendSet := stateStore.GetListenerDefaultBackendSet(port)
 		err = c.client.GetLbClient().CreateListener(context.TODO(), lbId, int(port), protocol, defaultBackendSet, listenerSslConfig)
@@ -482,6 +513,7 @@ func syncListener(namespace string, stateStore *state.StateStore, lbId *string,
 
 	needsUpdate := false
 	artifact, artifactType := stateStore.GetTLSConfigForListener(int32(*listener.Port))
+
 	var sslConfig *ociloadbalancer.SslConfigurationDetails
 	if artifact != "" {
 		sslConfig, err = GetSSLConfigForListener(namespace, &listener, artifactType, artifact, c.defaultCompartmentId, c.client)
@@ -495,6 +527,41 @@ func syncListener(namespace string, stateStore *state.StateStore, lbId *string,
 				needsUpdate = true
 			}
 		}
+
+		var port = int32(*listener.Port)
+		mode, deepth, trustcacert := stateStore.GetMutualTlsPortConfigForListener(port)
+		mtlsPorts := stateStore.IngressGroupState.MtlsPorts
+		klog.Infof("  syncListenerr  mtlsPorts :  %s ", util.PrettyPrint(mtlsPorts))
+
+		klog.Infof("  syncListener   before :  %s ", util.PrettyPrint(listener.SslConfiguration))
+		var modeToBool bool = false
+		if mode == util.MutualTlsAuthenticationVerify {
+			modeToBool = true
+		}
+		if *listener.SslConfiguration.VerifyPeerCertificate != modeToBool || *listener.SslConfiguration.VerifyDepth != deepth {
+			klog.Infof(" mtls port config %d needs update  mode:  %s", *listener.Name, mode)
+			needsUpdate = true
+
+			// check wethear the trustcacert is valid ca bundle
+			if mode == util.MutualTlsAuthenticationVerify && isTrustAuthorityCaBundle(trustcacert) {
+				listener.SslConfiguration.VerifyPeerCertificate = common.Bool(true)
+				listener.SslConfiguration.VerifyDepth = &deepth
+				caBundleIds := []string{trustcacert}
+				listener.SslConfiguration.TrustedCertificateAuthorityIds = caBundleIds
+
+			} else {
+				if mode == util.MutualTlsAuthenticationVerify {
+					klog.Error("verify trustcacert error, not a valid CA bundle ID: %s", util.PrettyPrint(trustcacert))
+				}
+				listener.SslConfiguration.VerifyPeerCertificate = common.Bool(false)
+				listener.SslConfiguration.VerifyDepth = common.Int(1)
+				listener.SslConfiguration.TrustedCertificateAuthorityIds = nil
+
+			}
+		}
+
+		klog.Infof("  syncListener  after :  %s ", util.PrettyPrint(listener.SslConfiguration))
+
 	}
 
 	protocol := stateStore.GetListenerProtocol(int32(*listener.Port))

pkg/controllers/ingress/util.go

@@ -187,6 +187,8 @@ func CreateCaBundle(certificateName string, compartmentId string, certificatesCl
 		CreateCaBundleDetails: caBundleDetails,
 		OpcRetryToken:         &certificateName,
 	}
+	klog.Infof("  createCaBundleRequest  ********** certificateName %s createCaBundleRequest :  %s ", certificateName, util.PrettyPrint(createCaBundleRequest))
+
 	createCaBundle, err := certificatesClient.CreateCaBundle(context.TODO(), createCaBundleRequest)
 	if err != nil {
 		return nil, err
@@ -373,7 +375,9 @@ func GetSSLConfigForListener(namespace string, listener *ociloadbalancer.Listene
 		}
 		newCertificateId = *cId
 	}
-
+	// //TODO add VerifyPeerCertificate here
+	//   https://github.com/oracle/oci-go-sdk/blob/master/example/example_loadbalancer_test.go
+	// listenerSslConfig.VerifyPeerCertificate(common.Bool(true))
 	if newCertificateId != "" {
 		certificateIds := []string{newCertificateId}
 		listenerSslConfig = &ociloadbalancer.SslConfigurationDetails{CertificateIds: certificateIds}

pkg/loadbalancer/loadbalancer.go

@@ -584,6 +584,10 @@ func (lbc *LoadBalancerClient) UpdateListener(ctx context.Context, lbId *string,
 	if sslConfigurationDetails == nil && l.SslConfiguration != nil {
 		sslConfigurationDetails = &loadbalancer.SslConfigurationDetails{
 			CertificateIds: l.SslConfiguration.CertificateIds,
+			//add for mtls verify
+			VerifyDepth:                    l.SslConfiguration.VerifyDepth,
+			VerifyPeerCertificate:          l.SslConfiguration.VerifyPeerCertificate,
+			TrustedCertificateAuthorityIds: l.SslConfiguration.TrustedCertificateAuthorityIds,
 		}
 	}
 

pkg/state/ingressstate.go

@@ -10,6 +10,7 @@
 package state
 
 import (
+	"encoding/json"
 	"fmt"
 	"reflect"
 
@@ -40,7 +41,12 @@ type TlsConfig struct {
 	Artifact string
 	Type     string
 }
-
+type MutualTlsPortConfig struct {
+	Port        int32  `json:"port"`
+	Mode        string `json:"mode"`
+	Depth       int    `json:"depth,omitempty"`
+	TrustCACert string `json:"trustcacert"`
+}
 type StateStore struct {
 	IngressClassLister networkinglisters.IngressClassLister
 	IngressLister      networkinglisters.IngressLister
@@ -58,6 +64,7 @@ type IngressClassState struct {
 	Listeners                  sets.Int32
 	ListenerProtocolMap        map[int32]string
 	ListenerTLSConfigMap       map[int32]TlsConfig
+	MtlsPorts                  map[int32]MutualTlsPortConfig
 	ListenerDefaultBsMap       map[int32]string
 }
 
@@ -108,13 +115,34 @@ func (s *StateStore) BuildState(ingressClass *networkingv1.IngressClass) error {
 	allBackendSets := sets.NewString(util.DefaultBackendSetName)
 	allListeners := sets.NewInt32()
 
+	// add mtls verify configmap
+	mutualTlsPortConfigMap := make(map[int32]MutualTlsPortConfig)
+
+	bsHealthCheckerMap[DefaultIngressName] = util.GetDefaultHeathChecker()
+	bsPolicyMap[DefaultIngressName] = util.DefaultBackendSetRoutingPolicy
+
 	bsHealthCheckerMap[util.DefaultBackendSetName] = util.GetDefaultHeathChecker()
 	bsPolicyMap[util.DefaultBackendSetName] = util.DefaultBackendSetRoutingPolicy
 
 	for _, ing := range ingressGroup {
 		hostSecretMap := make(map[string]string)
 		tlsConfiguredHosts := sets.NewString()
 		desiredPorts := sets.NewInt32()
+
+		validateMtlsPortAnnotationJson, err := ParseMutualTlsAnnotationJSON(ing)
+		klog.Infof("Ingress name: %s, validateMtlsPortAnnotationJson  %s", util.PrettyPrint(ing.Name), util.PrettyPrint(validateMtlsPortAnnotationJson))
+
+		if err != nil {
+			klog.Infof("Error parsing validateMtlsPortAnnotationJson JSON:", err)
+			return nil
+		}
+
+		for _, configPort := range validateMtlsPortAnnotationJson {
+			// add new mtls port to map
+			mutualTlsPortConfigMap[configPort.Port] = configPort
+		}
+		klog.Infof(" mutualTlsPortConfigMap  %s, mutualTlsPortConfigMap  %s", util.PrettyPrint(ing.Name), util.PrettyPrint(mutualTlsPortConfigMap))
+
 		// we always expect the default_ingress backendset
 		desiredBackendSets := sets.NewString(util.DefaultBackendSetName)
 
@@ -139,6 +167,10 @@ func (s *StateStore) BuildState(ingressClass *networkingv1.IngressClass) error {
 					return errors.Wrap(err, "error finding service and port")
 				}
 
+
+				desiredPorts.Insert(servicePort)
+				allListeners.Insert(servicePort)
+
 				listenerPort, err := util.DetermineListenerPort(ing, &tlsConfiguredHosts, host, servicePort)
 				if err != nil {
 					return errors.Wrap(err, "error determining listener port")
@@ -192,6 +224,8 @@ func (s *StateStore) BuildState(ingressClass *networkingv1.IngressClass) error {
 		Listeners:                  allListeners,
 		ListenerProtocolMap:        listenerProtocolMap,
 		ListenerTLSConfigMap:       listenerTLSConfigMap,
+		//add mutual tls port configmap
+		MtlsPorts: mutualTlsPortConfigMap,
 		ListenerDefaultBsMap:       listenerDefaultBsMap,
 	}
 
@@ -353,6 +387,22 @@ func (s *StateStore) GetTLSConfigForListener(port int32) (string, string) {
 	return "", ""
 }
 
+// func (s *StateStore) GetMutualTlsPortConfigForListener(port int32) MutualTlsPortConfig {
+// 	portMtlsConfig, ok := s.IngressGroupState.MtlsPorts[port]
+// 	if ok {
+// 		return portMtlsConfig
+// 	}
+// 	return MutualTlsPortConfig{}
+// }
+
+func (s *StateStore) GetMutualTlsPortConfigForListener(port int32) (string, int, string) {
+	portMtlsConfig, ok := s.IngressGroupState.MtlsPorts[port]
+	if ok {
+		return portMtlsConfig.Mode, portMtlsConfig.Depth, portMtlsConfig.TrustCACert
+	}
+	return "", 0, ""
+}
+
 func (s *StateStore) GetTLSConfigForBackendSet(bsName string) (string, string) {
 	bsTLSConfig, ok := s.IngressGroupState.BackendSetTLSConfigMap[bsName]
 	if ok {
@@ -380,3 +430,42 @@ func validatePortInUse(listenerTLSConfig TlsConfig, secretName string, certifica
 	}
 	return nil
 }
+
+func ParseMutualTlsAnnotationJSON(ing *networkingv1.Ingress) ([]MutualTlsPortConfig, error) {
+
+	mtlsPortsAnnotation := util.GetMutualTlsVerifyAnnotation(ing)
+	klog.Infof("Ingress name ParseMutualTlsAnnotationJSON %s, mtlsPortsAnnotation  %s", util.PrettyPrint(ing.Name), util.PrettyPrint(mtlsPortsAnnotation))
+
+	s := mtlsPortsAnnotation
+	//check if s is empty
+	if s == "" {
+		return []MutualTlsPortConfig{}, nil
+	}
+	// Check if the string conforms to JSON format
+	if !isValidJSON(s) {
+		return nil, fmt.Errorf("Original string does not conform to JSON format")
+	}
+
+	// Parse JSON string
+	var mtls []MutualTlsPortConfig
+	err := json.Unmarshal([]byte(s), &mtls)
+	if err != nil {
+		return nil, err
+	}
+
+	// Remove sub-objects without the 'port' field
+	var validMtls []MutualTlsPortConfig
+	for _, config := range mtls {
+		if config.Port != 0 {
+			validMtls = append(validMtls, config)
+		}
+	}
+
+	return validMtls, nil
+}
+
+// Check if the string conforms to JSON format
+func isValidJSON(s string) bool {
+	var js json.RawMessage
+	return json.Unmarshal([]byte(s), &js) == nil
+}

pkg/state/ingressstate_test.go

@@ -36,6 +36,7 @@ const (
 	TestIngressStateWithPortNameFilePath      = "test-ingress-state_withportname.yaml"
 	TestIngressStateWithNamedClassesFilePath  = "test-ingress-state_withnamedclasses.yaml"
 	TestSslTerminationAtLb                    = "test-ssl-termination-lb.yaml"
+	TestMtlsAuthVerfify                       = "validate-mutual-tls-authentication.yaml"
 	DefaultBackendSetValidationsFilePath      = "validate-default-backend-set.yaml"
 )
 
@@ -451,20 +452,43 @@ func TestSslTerminationAtLB(t *testing.T) {
 	Expect(lstTlsConfig.Type).Should(Equal(ArtifactTypeCertificate))
 }
 
-func TestValidateListenerDefaultBackendSet(t *testing.T) {
+
+func TestMtlsAuthVerifyPortConfig(t *testing.T) {
 	RegisterTestingT(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	ingressClassList := testutil.GetIngressClassList()
+
+	ingressList := testutil.ReadResourceAsIngressList(TestMtlsAuthVerfify)
+
+	testService := testutil.GetServiceListResource("default", "mtls-auth-verify-annotation", 943)
+	ingressClassLister, ingressLister, serviceLister := setUp(ctx, ingressClassList, ingressList, testService)
+
+	stateStore := NewStateStore(ingressClassLister, ingressLister, serviceLister, nil)
+	err := stateStore.BuildState(&ingressClassList.Items[0])
+	Expect(err).NotTo(HaveOccurred())
+
+	mode, deepth, _ := stateStore.GetMutualTlsPortConfigForListener(943)
+
+	Expect(mode).Should(Equal(util.MutualTlsAuthenticationVerify))
+	Expect(deepth).Should(Equal(1))
+
+}
+func TestValidateListenerDefaultBackendSet(t *testing.T) {
 	RegisterTestingT(t)
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	ingressClassList := testutil.GetIngressClassList()
 
+
 	ingressList := testutil.ReadResourceAsIngressList(DefaultBackendSetValidationsFilePath)
 
 	testService := testutil.GetServiceListResource("default", "tcp-test", 8080)
 	ingressClassLister, ingressLister, serviceLister := setUp(ctx, ingressClassList, ingressList, testService)
 
 	stateStore := NewStateStore(ingressClassLister, ingressLister, serviceLister, nil)
 	err := stateStore.BuildState(&ingressClassList.Items[0])
+
 	Expect(err).ShouldNot(HaveOccurred())
 
 	bsName := util.GenerateBackendSetName("default", "host-es", 8080)

pkg/state/validate-mutual-tls-authentication.yaml

@@ -0,0 +1,24 @@
+#
+# OCI Native Ingress Controller
+#
+# Copyright (c) 2023 Oracle America, Inc. and its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+#
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: ingress-mutual-authentication-annotation-one
+  namespace: default
+  annotations:
+    oci-native-ingress.oraclecloud.com/mutual-tls-authentication: '[{"port": 80, "mode": "passthrough"}, {"port": 943, "mode": "verify","depth":1 ,"trustCACert" : "ocid1.cabundle.oc1.phx.amaaaaaacuco5yqaiwnnqo54ffsumwoxjxtgwtvyyau3dv7gyeisykfavzta" }]'
+spec:
+  rules:
+  - http:
+      paths:
+      - pathType: Exact
+        path: "/HCPath"
+        backend:
+          service:
+            name: mtls-auth-verify-annotation
+            port:
+              number: 443

pkg/util/util.go

@@ -53,6 +53,12 @@ const (
 	IngressListenerTlsCertificateAnnotation = "oci-native-ingress.oraclecloud.com/certificate-ocid"
 	IngressBackendTlsEnabledAnnotation      = "oci-native-ingress.oraclecloud.com/backend-tls-enabled"
 
+	//verify client certificate , only use the lisentner bound CA certificate to verify the client certificate .
+	/*
+	 mutual-tls-authentication: '[{"port": 80, "mode": "passthrough"}, {"port": 443, "mode": "verify","depth":1 }]'
+	*/
+	IngressListenerMutualTlsVerifyAnnotation = "oci-native-ingress.oraclecloud.com/mutual-tls-authentication"
+
 	// IngressProtocolAnntoation - HTTP only for now
 	// HTTP, HTTP2, TCP - accepted.
 	IngressProtocolAnnotation = "oci-native-ingress.oraclecloud.com/protocol"
@@ -90,6 +96,8 @@ const (
 	CertificateCacheMaxAgeInMinutes = 10
 	LBCacheMaxAgeInMinutes          = 1
 	WAFCacheMaxAgeInMinutes         = 5
+
+	MutualTlsAuthenticationVerify = "verify"
 )
 
 var ErrIngressClassNotReady = errors.New("ingress class not ready")
@@ -642,6 +650,15 @@ func RetrievePods(endpointLister corelisters.EndpointsLister, podLister corelist
 	return pods, nil
 }
 
+
+func GetMutualTlsVerifyAnnotation(i *networkingv1.Ingress) string {
+	mtlsVerifyPorts, ok := i.Annotations[IngressListenerMutualTlsVerifyAnnotation]
+	if !ok {
+		return ""
+	}
+
+	return strings.ToLower(mtlsVerifyPorts)
+}
 func DetermineListenerPort(ingress *networkingv1.Ingress, tlsConfiguredHosts *sets.String, host string, servicePort int32) (int32, error) {
 	annotatedHttpPort, err := GetIngressHttpListenerPort(ingress)
 	if err != nil {