feat: OCI repository support for Helm charts (#207)

* feat: handle Helm charts in OCI repositories
* fix: write CA certs to system store; perms fudging
---------

Signed-off-by: Tyler Gillson <tyler.gillson@gmail.com>

Dockerfile

@@ -31,12 +31,19 @@ COPY pkg/ pkg/
 # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
 RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go
 
+RUN chmod 777 /etc /etc/ssl && chmod -R 777 /etc/ssl/certs && \
+    mkdir /charts && chmod -R 777 /charts
+
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM --platform=$TARGETPLATFORM gcr.io/distroless/static:nonroot AS production
 WORKDIR /
+
 COPY --from=builder /workspace/manager .
 COPY --from=builder /workspace/helm .
+COPY --from=builder --chown=65532:65532 /etc /etc
+COPY --from=builder --chown=65532:65532 /charts /charts
+
 USER 65532:65532
 
 ENTRYPOINT ["/manager"]

api/v1alpha1/validatorconfig_types.go

@@ -86,7 +86,7 @@ type ValidatorPluginCondition struct {
 // ConditionType is a valid value for Condition.Type.
 type ConditionType string
 
-// HelmChartDeployedCondition defines the helm chart deployed condition type that defines if the helm chart was deployed correctly.
+// HelmChartDeployedCondition defines whether the helm chart was installed/pulled/upgraded correctly.
 const HelmChartDeployedCondition ConditionType = "HelmChartDeployed"
 
 //+kubebuilder:object:root=true

internal/controller/suite_test.go

@@ -45,7 +45,7 @@ import (
 	"github.com/spectrocloud-labs/validator/internal/kube"
 	"github.com/spectrocloud-labs/validator/internal/sinks"
 	"github.com/spectrocloud-labs/validator/pkg/helm"
-	"github.com/spectrocloud-labs/validator/pkg/util/ptr"
+	"github.com/spectrocloud-labs/validator/pkg/util"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -95,11 +95,11 @@ var _ = BeforeSuite(func() {
 		BinaryAssetsDirectory: filepath.Join(
 			"..", "..", "bin", "k8s", fmt.Sprintf("%s-%s-%s", k8sVersion, runtime.GOOS, runtime.GOARCH),
 		),
-		UseExistingCluster: ptr.Ptr(false),
+		UseExistingCluster: util.Ptr(false),
 	}
 
 	if os.Getenv("KUBECONFIG") != "" {
-		testEnv.UseExistingCluster = ptr.Ptr(true)
+		testEnv.UseExistingCluster = util.Ptr(true)
 	}
 
 	// monkey-patch binary paths

internal/controller/validatorconfig_controller.go

@@ -53,8 +53,6 @@ const (
 
 	// An annotation added to a ValidatorConfig to determine whether or not to update a plugin's Helm release
 	PluginValuesHash = "validator/plugin-values"
-
-	helmCAFile = "/tmp/ca.crt"
 )
 
 var (
@@ -198,7 +196,7 @@ func (r *ValidatorConfigReconciler) redeployIfNeeded(ctx context.Context, vc *v1
 			continue
 		}
 
-		upgradeOpts := &helm.UpgradeOptions{
+		opts := &helm.Options{
 			Chart:                 p.Chart.Name,
 			Repo:                  p.Chart.Repository,
 			Version:               p.Chart.Version,
@@ -208,16 +206,35 @@ func (r *ValidatorConfigReconciler) redeployIfNeeded(ctx context.Context, vc *v1
 
 		if p.Chart.AuthSecretName != "" {
 			nn := types.NamespacedName{Name: p.Chart.AuthSecretName, Namespace: vc.Namespace}
-			if err := r.configureHelmOpts(nn, upgradeOpts); err != nil {
+			if err := r.configureHelmOpts(nn, opts); err != nil {
 				r.Log.V(0).Error(err, "failed to configure basic auth for Helm upgrade")
 				conditions[i] = r.buildHelmChartCondition(p.Chart.Name, err)
 				continue
 			}
 		}
 
-		r.Log.V(0).Info("Installing/upgrading plugin Helm chart", "namespace", vc.Namespace, "name", p.Chart.Name)
+		var cleanupLocalChart bool
+		if strings.HasPrefix(p.Chart.Repository, "oci://") {
+			r.Log.V(0).Info("Pulling plugin Helm chart", "name", p.Chart.Name)
+
+			opts.Untar = true
+			opts.UntarDir = "/charts"
+			opts.Version = strings.TrimPrefix(opts.Version, "v")
+
+			if err := r.HelmClient.Pull(*opts); err != nil {
+				r.Log.V(0).Error(err, "failed to pull Helm chart from OCI repository")
+				conditions[i] = r.buildHelmChartCondition(p.Chart.Name, err)
+				continue
+			}
 
-		err := r.HelmClient.Upgrade(p.Chart.Name, vc.Namespace, *upgradeOpts)
+			r.Log.V(0).Info("Reconfiguring Helm options to deploy local chart", "name", p.Chart.Name)
+			opts.Path = fmt.Sprintf("/charts/%s", opts.Chart)
+			opts.Chart = ""
+			cleanupLocalChart = true
+		}
+
+		r.Log.V(0).Info("Installing/upgrading plugin Helm chart", "namespace", vc.Namespace, "name", p.Chart.Name)
+		err := r.HelmClient.Upgrade(p.Chart.Name, vc.Namespace, *opts)
 		if err != nil {
 			// if Helm install/upgrade failed, delete the release so installation is reattempted each iteration
 			if strings.Contains(err.Error(), "has no deployed releases") {
@@ -227,6 +244,13 @@ func (r *ValidatorConfigReconciler) redeployIfNeeded(ctx context.Context, vc *v1
 			}
 		}
 		conditions[i] = r.buildHelmChartCondition(p.Chart.Name, err)
+
+		if cleanupLocalChart {
+			r.Log.V(0).Info("Cleaning up local chart directory", "path", opts.Path)
+			if err := os.RemoveAll(opts.Path); err != nil {
+				r.Log.V(0).Error(err, "failed to remove local chart directory")
+			}
+		}
 	}
 
 	// delete any plugins that have been removed
@@ -242,7 +266,7 @@ func (r *ValidatorConfigReconciler) redeployIfNeeded(ctx context.Context, vc *v1
 	return nil
 }
 
-func (r *ValidatorConfigReconciler) configureHelmOpts(nn types.NamespacedName, opts *helm.UpgradeOptions) error {
+func (r *ValidatorConfigReconciler) configureHelmOpts(nn types.NamespacedName, opts *helm.Options) error {
 	secret := &corev1.Secret{}
 	if err := r.Get(context.TODO(), nn, secret); err != nil {
 		return fmt.Errorf(
@@ -265,10 +289,11 @@ func (r *ValidatorConfigReconciler) configureHelmOpts(nn types.NamespacedName, o
 
 	caCert, ok := secret.Data["caCert"]
 	if ok {
-		if err := os.WriteFile(helmCAFile, caCert, 0600); err != nil {
+		caFile := fmt.Sprintf("/etc/ssl/certs/%s-ca.crt", opts.Chart)
+		if err := os.WriteFile(caFile, caCert, 0600); err != nil {
 			return wrapErrors.Wrap(err, "failed to write Helm CA file")
 		}
-		opts.CaFile = helmCAFile
+		opts.CaFile = caFile
 	}
 
 	return nil
@@ -337,7 +362,7 @@ func (r *ValidatorConfigReconciler) buildHelmChartCondition(chartName string, er
 		Type:               v1alpha1.HelmChartDeployedCondition,
 		PluginName:         chartName,
 		Status:             corev1.ConditionTrue,
-		Message:            fmt.Sprintf("Plugin %s is installed", chartName),
+		Message:            fmt.Sprintf("Plugin chart %s is installed/upgraded", chartName),
 		LastTransitionTime: metav1.Time{Time: time.Now()},
 	}
 	if err != nil {

internal/controller/validatorconfig_controller_test.go

@@ -178,7 +178,7 @@ func TestConfigureHelmBasicAuth(t *testing.T) {
 		name       string
 		reconciler ValidatorConfigReconciler
 		nn         types.NamespacedName
-		opts       *helm.UpgradeOptions
+		opts       *helm.Options
 		expected   error
 	}{
 		{
@@ -189,7 +189,7 @@ func TestConfigureHelmBasicAuth(t *testing.T) {
 					GetErrors: []error{errors.New("get failed")},
 				},
 			},
-			opts:     &helm.UpgradeOptions{Chart: "foo", Repo: "bar"},
+			opts:     &helm.Options{Chart: "foo", Repo: "bar"},
 			expected: errors.New("failed to get auth secret chart-secret in namespace validator for chart foo in repo bar: get failed"),
 		},
 		{
@@ -198,7 +198,7 @@ func TestConfigureHelmBasicAuth(t *testing.T) {
 			reconciler: ValidatorConfigReconciler{
 				Client: test.ClientMock{},
 			},
-			opts:     &helm.UpgradeOptions{Chart: "foo", Repo: "bar"},
+			opts:     &helm.Options{Chart: "foo", Repo: "bar"},
 			expected: errors.New("auth secret for chart foo in repo bar missing required key: 'username'"),
 		},
 	}

pkg/helm/helm.go

@@ -14,12 +14,22 @@ import (
 	klog "k8s.io/klog/v2"
 )
 
-var CommandPath = "./helm"
+var (
+	CommandPath   = "./helm"
+	preserveFiles = false // whether to preserve kubeconfig and Helm values files
+)
+
+func init() {
+	if os.Getenv("HELM_PRESERVE_FILES") == "true" {
+		preserveFiles = true
+	}
+}
 
-// HelmClient defines the interface how to interact with Helm
+// HelmClient is an interface for interacting with Helm
 type HelmClient interface {
-	Upgrade(name, namespace string, options UpgradeOptions) error
 	Delete(name, namespace string) error
+	Pull(options Options) error
+	Upgrade(name, namespace string, options Options) error
 }
 
 type helmClient struct {
@@ -43,86 +53,52 @@ func (c *helmClient) Delete(name, namespace string) error {
 	if err != nil {
 		return err
 	}
-	defer os.Remove(kubeConfig)
+	if !preserveFiles {
+		defer os.Remove(kubeConfig)
+	}
 
 	args := []string{"delete", name, "--namespace", namespace, "--kubeconfig", kubeConfig}
 	return c.exec(args)
 }
 
-func (c *helmClient) Upgrade(name, namespace string, options UpgradeOptions) error {
-	options.ExtraArgs = append(options.ExtraArgs, "--install")
-	return c.run(name, namespace, options, "upgrade", options.ExtraArgs)
-}
-
-func (c *helmClient) exec(args []string) error {
-	if len(args) == 0 {
-		return nil
-	}
-
-	sb := strings.Builder{}
-	mask := false
-	for _, a := range args {
-		if mask {
-			sb.WriteString("***** ")
-			mask = false
-			continue
-		}
-		if a == "--password" {
-			mask = true
-		}
-		sb.WriteString(a)
-		sb.WriteString(" ")
-	}
-	sanitizedArgs := sb.String()
-
-	fmt.Println("helm " + sanitizedArgs)
-	cmd := exec.Command(c.helmPath, args...) // #nosec G204
-	if c.stdout != nil {
-		cmd.Stdout = c.stdout
-		cmd.Stderr = c.stderr
-		return cmd.Run()
-	}
-
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		if strings.Contains(string(output), "release: not found") {
-			return nil
-		}
-		klog.Errorf("Error executing command: helm %s", sanitizedArgs)
-		klog.Errorf("Output: %s, Error: %v", string(output), err)
-		return fmt.Errorf("error executing helm %s: %s", args[0], string(output))
+func (c *helmClient) Pull(options Options) error {
+	if options.Repo == "" {
+		return fmt.Errorf("chart repo cannot be null")
 	}
+	args := []string{"pull", options.Repo}
+	args = options.ConfigureVersion(args)
+	args = options.ConfigureArchive(args)
+	args = options.ConfigureAuth(args)
+	args = options.ConfigureTLS(args)
+	return c.exec(args)
+}
 
-	return nil
+func (c *helmClient) Upgrade(name, namespace string, options Options) error {
+	options.ExtraArgs = append(options.ExtraArgs, "--install")
+	return c.run(name, namespace, options, "upgrade", options.ExtraArgs)
 }
 
-func (c *helmClient) run(name, namespace string, options UpgradeOptions, command string, extraArgs []string) error {
+func (c *helmClient) run(name, namespace string, options Options, command string, extraArgs []string) error {
 	kubeConfig, err := writeKubeConfig(c.config)
 	if err != nil {
 		return err
 	}
-	defer os.Remove(kubeConfig)
+	if !preserveFiles {
+		defer os.Remove(kubeConfig)
+	}
 
 	args := []string{command, name}
 	if options.Path != "" {
 		args = append(args, options.Path)
 	} else if options.Chart != "" {
 		args = append(args, options.Chart)
-
 		if options.Repo == "" {
 			return fmt.Errorf("chart repo cannot be null")
 		}
-
-		args = append(args, "--repo", options.Repo)
-		if options.Version != "" {
-			args = append(args, "--version", options.Version)
-		}
-		if options.Username != "" {
-			args = append(args, "--username", options.Username)
-		}
-		if options.Password != "" {
-			args = append(args, "--password", options.Password)
-		}
+		args = options.ConfigureRepo(args)
+		args = options.ConfigureVersion(args)
+		args = options.ConfigureAuth(args)
+		args = options.ConfigureTLS(args)
 	}
 
 	args = append(args, "--kubeconfig", kubeConfig, "--namespace", namespace)
@@ -152,7 +128,9 @@ func (c *helmClient) run(name, namespace string, options UpgradeOptions, command
 		if err := tempFile.Close(); err != nil {
 			return errors.Wrap(err, "close temp file")
 		}
-		defer os.Remove(tempFile.Name())
+		if !preserveFiles {
+			defer os.Remove(tempFile.Name())
+		}
 
 		// Wait quickly so helm will find the file
 		time.Sleep(time.Millisecond)
@@ -197,15 +175,49 @@ func (c *helmClient) run(name, namespace string, options UpgradeOptions, command
 		args = append(args, "--atomic")
 	}
 
-	// TLS options
-	if options.CaFile != "" {
-		args = append(args, "--ca-file", options.CaFile)
+	return c.exec(args)
+}
+
+func (c *helmClient) exec(args []string) error {
+	if len(args) == 0 {
+		return nil
 	}
-	if options.InsecureSkipTlsVerify {
-		args = append(args, "--insecure-skip-tls-verify")
+
+	sb := strings.Builder{}
+	mask := false
+	for _, a := range args {
+		if mask {
+			sb.WriteString("***** ")
+			mask = false
+			continue
+		}
+		if a == "--password" {
+			mask = true
+		}
+		sb.WriteString(a)
+		sb.WriteString(" ")
 	}
+	sanitizedArgs := sb.String()
 
-	return c.exec(args)
+	fmt.Println("helm " + sanitizedArgs)
+	cmd := exec.Command(c.helmPath, args...) // #nosec G204
+	if c.stdout != nil {
+		cmd.Stdout = c.stdout
+		cmd.Stderr = c.stderr
+		return cmd.Run()
+	}
+
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		if strings.Contains(string(output), "release: not found") {
+			return nil
+		}
+		klog.Errorf("Error executing command: helm %s", sanitizedArgs)
+		klog.Errorf("Output: %s, Error: %v", string(output), err)
+		return fmt.Errorf("error executing helm %s: %s", args[0], string(output))
+	}
+
+	return nil
 }
 
 // writeKubeConfig writes the kubeconfig to a file and returns the filename