Certificate support for image registry

Remove the InsecureSkipTLSVerify annotations.

* Create a ClusterIssuer CA (via openssl) that is used by OLMv1 e2e
* Update the operator controller to specify a cert directory, rather than a
  single file.
* Use this directory for catalogd and image-registries
* Update the deployment to reference CAs appropriately

Signed-off-by: Todd Short <tshort@redhat.com>

cmd/manager/main.go

@@ -79,11 +79,11 @@ func main() {
 		cachePath                 string
 		operatorControllerVersion bool
 		systemNamespace           string
-		caCert                    string
+		caCertDir                 string
 	)
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
-	flag.StringVar(&caCert, "ca-cert", "", "The TLS certificate to use for verifying HTTPS connections to the Catalogd web server.")
+	flag.StringVar(&caCertDir, "ca-cert", "", "The directory of TLS certificate to use for verifying HTTPS connections to the Catalogd and docker-registry web servers.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
@@ -151,7 +151,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	httpClient, err := httputil.BuildHTTPClient(caCert)
+	httpClient, err := httputil.BuildHTTPClient(caCertDir)
 	if err != nil {
 		setupLog.Error(err, "unable to create catalogd http client")
 	}
@@ -217,6 +217,7 @@ func main() {
 		InstalledBundleGetter: &controllers.DefaultInstalledBundleGetter{ActionClientGetter: acg},
 		Handler:               registryv1handler.HandlerFunc(registry.HandleBundleDeployment),
 		Finalizers:            clusterExtensionFinalizers,
+		CaCertDir:             caCertDir,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "ClusterExtension")
 		os.Exit(1)

config/overlays/e2e/kustomization.yaml

@@ -4,6 +4,8 @@ resources:
 - ../../overlays/tls
 - manager_e2e_coverage_pvc.yaml
 - manager_e2e_coverage_copy_pod.yaml
+- manager_e2e_cert_certs.yaml
 
 patches:
 - path: manager_e2e_coverage_patch.yaml
+  path: manager_e2e_cert_patch.yaml

config/overlays/e2e/manager_e2e_cert_certs.yaml

@@ -0,0 +1,24 @@
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: olmv1-ca
+spec:
+  ca:
+    secretName: olmv1-ca
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: olmv1-cert
+spec:
+  secretName: olmv1-cert
+  dnsNames:
+    - operator-controller.olmv1-system.svc
+    - operator-controller.olmv1-system.svc.cluster.local
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: olmv1-ca
+    kind: ClusterIssuer
+    group: cert-manager.io

config/overlays/e2e/manager_e2e_cert_patch.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: kube-rbac-proxy
+      - name: manager
+        volumeMounts:
+        - name: e2e-cert
+          mountPath: /var/certs/olm-ca.crt
+          subPath: olm-ca.crt
+          readOnly: true
+      volumes:
+      - name: e2e-cert
+        secret:
+          secretName: olmv1-cert
+          items:
+            - key: ca.crt
+              path: olm-ca.crt

config/overlays/tls/patches/manager_deployment_cert.yaml

@@ -1,9 +1,9 @@
 - op: add
   path: /spec/template/spec/volumes/-
-  value: {"name":"ca-certificate", "secret":{"secretName":"catalogd-catalogserver-cert", "optional": false, "items": [{"key": "tls.crt", "path": "tls.crt"}]}}
+  value: {"name":"catalogd-certificate", "secret":{"secretName":"catalogd-catalogserver-cert", "optional": false, "items": [{"key": "ca.crt", "path": "catalogd.crt"}]}}
 - op: add
   path: /spec/template/spec/containers/0/volumeMounts/-
-  value: {"name":"ca-certificate", "readOnly": true, "mountPath":"/var/certs"}
+  value: {"name":"catalogd-certificate", "readOnly": true, "mountPath":"/var/certs/catalogd.crt", "subPath":"catalogd.crt"}
 - op: add
   path: /spec/template/spec/containers/0/args/-
-  value: "--ca-cert=/var/certs/tls.crt"
+  value: "--ca-cert=/var/certs"

internal/controllers/clusterextension_controller.go

@@ -22,6 +22,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"os"
+	"path/filepath"
 	"sort"
 	"strings"
 	"sync"
@@ -90,16 +92,13 @@ type ClusterExtensionReconciler struct {
 	cache                 cache.Cache
 	InstalledBundleGetter InstalledBundleGetter
 	Finalizers            crfinalizer.Finalizers
+	CaCertDir             string
 }
 
 type InstalledBundleGetter interface {
 	GetInstalledBundle(ctx context.Context, ext *ocv1alpha1.ClusterExtension) (*ocv1alpha1.BundleMetadata, error)
 }
 
-const (
-	bundleConnectionAnnotation string = "bundle.connection.config/insecureSkipTLSVerify"
-)
-
 //+kubebuilder:rbac:groups=olm.operatorframework.io,resources=clusterextensions,verbs=get;list;watch
 //+kubebuilder:rbac:groups=olm.operatorframework.io,resources=clusterextensions/status,verbs=update;patch
 //+kubebuilder:rbac:groups=olm.operatorframework.io,resources=clusterextensions/finalizers,verbs=update
@@ -249,7 +248,7 @@ func (r *ClusterExtensionReconciler) reconcile(ctx context.Context, ext *ocv1alp
 	// Generate a BundleDeployment from the ClusterExtension to Unpack.
 	// Note: The BundleDeployment here is not a k8s API, its a simple Go struct which
 	// necessary embedded values.
-	bd := r.generateBundleDeploymentForUnpack(bundle.Image, ext)
+	bd := r.generateBundleDeploymentForUnpack(ctx, bundle.Image, ext)
 	unpackResult, err := r.Unpacker.Unpack(ctx, bd)
 	if err != nil {
 		setStatusUnpackFailed(ext, err.Error())
@@ -533,7 +532,11 @@ func SetDeprecationStatus(ext *ocv1alpha1.ClusterExtension, bundle *catalogmetad
 	}
 }
 
-func (r *ClusterExtensionReconciler) generateBundleDeploymentForUnpack(bundlePath string, ce *ocv1alpha1.ClusterExtension) *rukpakv1alpha2.BundleDeployment {
+func (r *ClusterExtensionReconciler) generateBundleDeploymentForUnpack(ctx context.Context, bundlePath string, ce *ocv1alpha1.ClusterExtension) *rukpakv1alpha2.BundleDeployment {
+	certData, err := r.getCertificateData(ce)
+	if err != nil {
+		log.FromContext(ctx).WithName("operator-controller").WithValues("cluster-extension", ce.GetName()).Error(err, "unable to get TLS certificate")
+	}
 	return &rukpakv1alpha2.BundleDeployment{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       ce.Kind,
@@ -549,22 +552,35 @@ func (r *ClusterExtensionReconciler) generateBundleDeploymentForUnpack(bundlePat
 				Type: rukpakv1alpha2.SourceTypeImage,
 				Image: &rukpakv1alpha2.ImageSource{
 					Ref:                   bundlePath,
-					InsecureSkipTLSVerify: isInsecureSkipTLSVerifySet(ce),
+					InsecureSkipTLSVerify: false,
+					CertificateData:       certData,
 				},
 			},
 		},
 	}
 }
 
-func isInsecureSkipTLSVerifySet(ce *ocv1alpha1.ClusterExtension) bool {
-	if ce == nil {
-		return false
+func (r *ClusterExtensionReconciler) getCertificateData(ce *ocv1alpha1.ClusterExtension) (string, error) {
+	if r.CaCertDir == "" {
+		return "", nil
 	}
-	value, ok := ce.Annotations[bundleConnectionAnnotation]
-	if !ok {
-		return false
+
+	var certs []string
+	err := filepath.Walk(r.CaCertDir, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		certs = append(certs, string(data))
+		return nil
+	})
+	if err != nil {
+		return "", err
 	}
-	return value == "true"
+	return strings.Join(certs, "\n"), nil
 }
 
 // SetupWithManager sets up the controller with the Manager.

internal/httputil/httputil.go

@@ -5,30 +5,46 @@ import (
 	"crypto/x509"
 	"net/http"
 	"os"
+	"path/filepath"
+	"strings"
 	"time"
 )
 
-func BuildHTTPClient(caCert string) (*http.Client, error) {
+func BuildHTTPClient(caDir string) (*http.Client, error) {
 	httpClient := &http.Client{Timeout: 10 * time.Second}
 
-	if caCert != "" {
-		// tlsFileWatcher, err := certwatcher.New(caCert, "")
+	// use the SystemCertPool as a default
+	caCertPool, err := x509.SystemCertPool()
+	if err != nil {
+		return nil, err
+	}
 
-		cert, err := os.ReadFile(caCert)
+	if caDir != "" {
+		var certs []string
+		err := filepath.Walk(caDir, func(path string, info os.FileInfo, err error) error {
+			if info.IsDir() {
+				return nil
+			}
+			data, err := os.ReadFile(path)
+			if err != nil {
+				return err
+			}
+			certs = append(certs, string(data))
+			return nil
+		})
 		if err != nil {
 			return nil, err
 		}
-		caCertPool := x509.NewCertPool()
-		caCertPool.AppendCertsFromPEM(cert)
-		tlsConfig := &tls.Config{
-			RootCAs:    caCertPool,
-			MinVersion: tls.VersionTLS12,
-		}
-		tlsTransport := &http.Transport{
-			TLSClientConfig: tlsConfig,
-		}
-		httpClient.Transport = tlsTransport
+		caCertPool.AppendCertsFromPEM([]byte(strings.Join(certs, "\n")))
+	}
+	tlsConfig := &tls.Config{
+		RootCAs:    caCertPool,
+		MinVersion: tls.VersionTLS12,
+	}
+	tlsTransport := &http.Transport{
+		TLSClientConfig: tlsConfig,
 	}
+	httpClient.Transport = tlsTransport
 
 	return httpClient, nil
 }

scripts/install.tpl.sh

@@ -35,6 +35,49 @@ function kubectl_wait() {
 kubectl apply -f "https://github.com/cert-manager/cert-manager/releases/download/${cert_mgr_version}/cert-manager.yaml"
 kubectl_wait "cert-manager" "deployment/cert-manager-webhook" "60s"
 
+# Use a name from the environment if specified
+root_ca="${ROOT_CA:-root-ca}"
+
+# If the cert does not exist, create it
+if [ ! -e "${root_ca}.crt" ]; then
+    # Create a CA certificate for cert-manager via openssl
+    cat >${root_ca}.cnf <<EOF
+[default]
+name = ${root_ca}
+default_ca = ca_default
+
+[req]
+encrypt_key             = no
+default_md              = sha256
+utf8                    = yes
+string_mask             = utf8only
+prompt                  = no
+distinguished_name      = ca_dn
+req_extensions          = ca_ext
+
+[ca_dn]
+organizationName = operator-framework
+organizationalUnitName = operator-controller
+
+[ca_ext]
+basicConstraints        = critical,CA:true
+keyUsage                = critical,keyCertSign,cRLSign
+subjectKeyIdentifier    = hash
+EOF
+
+    openssl genrsa -out ${root_ca}.key 4096
+    openssl req -new -x509 -config ${root_ca}.cnf -key ${root_ca}.key -out ${root_ca}.crt -days 4800 -extensions ca_ext
+    function cleanup {
+        rm ${root_ca}.key
+        rm ${root_ca}.cnf
+        rm ${root_ca}.crt
+    }
+    trap cleanup EXIT
+fi
+
+# Define the "olmv1-ca" secret from those files
+kubectl create secret tls olmv1-ca -n cert-manager --cert=${root_ca}.crt --key=${root_ca}.key
+
 kubectl apply -f "https://github.com/operator-framework/catalogd/releases/download/${catalogd_version}/catalogd.yaml"
 kubectl_wait "olmv1-system" "deployment/catalogd-controller-manager" "60s"
 

test/e2e/cluster_extension_install_test.go

@@ -45,9 +45,6 @@ func testInit(t *testing.T) (*ocv1alpha1.ClusterExtension, *catalogd.ClusterCata
 	clusterExtension := &ocv1alpha1.ClusterExtension{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: clusterExtensionName,
-			Annotations: map[string]string{
-				"bundle.connection.config/insecureSkipTLSVerify": "true",
-			},
 		},
 	}
 	return clusterExtension, extensionCatalog

test/extension-developer-e2e/extension_developer_test.go

@@ -35,9 +35,6 @@ func TestExtensionDeveloper(t *testing.T) {
 		{
 			ObjectMeta: metav1.ObjectMeta{
 				Name: "registryv1",
-				Annotations: map[string]string{
-					"bundle.connection.config/insecureSkipTLSVerify": "true",
-				},
 			},
 			Spec: ocv1alpha1.ClusterExtensionSpec{
 				PackageName:      os.Getenv("REG_PKG_NAME"),

test/tools/image-registry.sh

@@ -30,14 +30,6 @@ metadata:
   name: ${namespace}
 ---
 apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
-  name: selfsigned-issuer
-  namespace: ${namespace}
-spec:
-  selfSigned: {}
----
-apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
   name: ${namespace}-registry
@@ -52,8 +44,8 @@ spec:
     algorithm: ECDSA
     size: 256
   issuerRef:
-    name: selfsigned-issuer
-    kind: Issuer
+    name: olmv1-ca
+    kind: ClusterIssuer
     group: cert-manager.io
 ---
 apiVersion: apps/v1