secrets feature

.vscode/settings.json

@@ -1,5 +1,6 @@
 {
     "yaml.schemas": {
-        "https://json.schemastore.org/github-workflow.json": "file:///Users/pthomison/Projects/k3auto/.github/workflows/goreleaser.yaml"
+        "https://json.schemastore.org/github-workflow.json": "file:///Users/pthomison/Projects/k3auto/.github/workflows/goreleaser.yaml",
+        "https://json.schemastore.org/yamllint.json": "file:///Users/pthomison/Projects/k3auto/internal/secrets/config.test.yaml"
     }
 }

README.md

@@ -1,6 +1,6 @@
 # K3Auto
 
-An golang CLI tool used for rapidly deploying kubernetes evironments in a repeatable manner for local testing
+An golang CLI tool used for rapidly deploying kubernetes environments in a repeatable manner for local testing
 
 Powered By:
 - [flux](https://fluxcd.io/)
@@ -18,8 +18,10 @@ brew install pthomison/tools/k3auto
 ### Deployment Order Of Operations
 1. Deploy k3d Cluster
 2. Inject Flux Controllers
-3. Deploy Embedded Deployments
-4. Deploy Runtime Deployments
+3. Inject Docker Registry
+4. Inject Secrets
+5. Deploy Embedded Deployments
+6. Deploy Runtime Deployments
 
 ### Options
 ```
@@ -42,6 +44,7 @@ Flags:
   -d, --deployment-directory string   Deployment Directory
   -h, --help                          help for k3auto
   -m, --minimal                       Only deploy the k3d cluster & flux controllers
+  -s, --secret-config string          Inject Secrets To the Cluster on Creation
 
 Use "k3auto [command] --help" for more information about a command.
 
@@ -57,13 +60,50 @@ By default, k3auto will deploy a single node k3d cluster and will inject the fol
 
 To *only* deploy a the k3d cluster and flux controllers, use the `--minimal`/`-m` flag.
 
-
 The default k3d cluster config file is embeded in the binary, but can be found at `k3auto/default/k3d-config.yaml`. To use a different config file at runtime, use the `--cluster-config`/`-c` flag.
 
 ### Deploying Resources
 
 To deploy your own desired resources at runtime, use the `--deployment-directory`/`-d` flag and supply yaml manifests within that directory. At present moment, k3auto will capture that directory into an OCI Image, ship the image to the k3d registry, then create flux OCIRepository & Kustomization objects that will deploy your manifests into your cluster. A kustomization.yaml can be supplied within this directory if any kustomization changes are desired.
 
+### Updating Resources
+
+After creating your environment, you can update the manifests inside your deployment directory and then run `k3auto update`. This will update the manifests inside the environment and then deploy your changes.
+
+
+### Secrets
+
+With local kubernetes clusters, the need inevitably arises to inject & manage secrets for passwords, cloud credentials, etc. Storing passwords in plaintext in a repo is a non-starter, so you could inject them by hand after the local environment is created but this quickly becomes cumbersome.
+
+Generally I prefer to use a secret store external to my environment (such as AWS Parameter Store), then use automation to inject that into my environment (like the wonderful [External Secrets](https://external-secrets.io/) project). This automation still requires bootstrapping however, so I still need a way to inject secrets from my local machine at runtime.
+
+To support this, k3auto uses a secrets config file that will support different secret backends and will automatically create these as secrets in your cluster at creation.
+
+Currently, the only secret backend that is written is a bash exec backend (this is to enable shelling out to other CLI tools such as the awscli or accessing the systems keyring). If tigheter integration with a given provider is required, a custom backend can be written trivialy, with the only interface function being `Resolve(ctx context.Context, args []string) (string, error)`.
+
+Example Secrets Config File
+```
+DefaultSecret: defaultSecretName
+DefaultNamespace: default
+Secrets:
+  - Type: exec
+    SecretName: "flux-system"
+    SecretKey: "known-hosts"
+    Args:
+      - /bin/bash
+      - -c
+      - "aws ssm get-parameter --name /lab/flux-known-hosts --with-decryption | jq -r '.Parameter.Value'"
+
+  - Type: exec
+    SecretName: "flux-system" # supports adding multiple keys to a single secret
+    SecretKey: "flux-public-key"
+    Args:
+      - /bin/bash
+      - -c
+      - "aws ssm get-parameter --name /lab/flux-public-key --with-decryption | jq -r '.Parameter.Value'"
+
+```
+
 
 ### Modifications
 
@@ -74,4 +114,15 @@ To embed your own deployment manifests, just fork this repository. Then add your
 1. ~~An `update` subcommand for refreshing user deployments~~ (Update now works!)
 2. ~~Better image solution for OCI deployments (network registry layer is a current weak point)~~ (Now registry lives within the cluster && k3auto port-forwards into the pod for connectivity)
 3. Standardize ingress solution
-4. Preloading base images to prevent excessive bandwidth consumption
+4. Preloading base images to prevent excessive bandwidth consumption
+5. Secrets Management (In Progress)
+6. Internal package testing
+7. Flux Reconciliation Requests
+
+
+### Project Learnings
+- Port-Forwarding from CLI apps is a little complication/not well documented, but is extremely useful for k8s environments in unknown network setups
+- Go makes glueing together a lot of different projects/tools ~~very~~ relatively easy, all dependencies between the projects will need to be compatible. Not too big of an issue for unrelated tools, but can cause some funkiness if they have similar domains (k3d and docker-cli being the combo I ran into on this project)
+- Garbage Collected Languages (ie golang) need/should/(good idea?) to invoke a non-GC lang for cryptographic key operations so it can release the key memory and not rely on GC to remove it. Still working on groking this one, but checkout https://github.com/containers/image/issues/1634 if interested
+- Executing `kubectl apply` like commands from client-go is possible (should be... since kubectl is written with it :wink: ), but the command does a lot of work under the covers & I have found it easier to just decode the objects (a la `runtime.Decoder`) and then use the standard client-go semantics.
+- When splitting yaml objects (specifically for CRDs) on `---`, be sure to use regex/start-of-line/end-of-line

cmd/create.go

@@ -2,13 +2,15 @@ package cmd
 
 import (
 	"context"
+	"os"
 
 	k3dv1alpha5 "github.com/k3d-io/k3d/v5/pkg/config/v1alpha5"
 	k3druntimes "github.com/k3d-io/k3d/v5/pkg/runtimes"
 	defaults "github.com/pthomison/k3auto/default"
 	"github.com/pthomison/k3auto/internal/flux"
 	"github.com/pthomison/k3auto/internal/k3d"
 	"github.com/pthomison/k3auto/internal/k8s"
+	"github.com/pthomison/k3auto/internal/secrets"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
@@ -84,6 +86,26 @@ func injectRegistry(ctx context.Context) error {
 	return nil
 }
 
+func injectSecrets(ctx context.Context, k8sC ctrlclient.Client) error {
+	f, err := os.Open(SecretConfigFileFlag)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	conf, err := secrets.LoadConfigFile(f)
+	if err != nil {
+		return err
+	}
+
+	err = secrets.InjectSecrets(ctx, k8sC, conf)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func k3AutoCreate(cmd *cobra.Command, args []string) {
 	ctx := cmd.Context()
 	if ctx == nil {
@@ -118,6 +140,11 @@ func k3AutoCreate(cmd *cobra.Command, args []string) {
 	})
 	checkError(err)
 
+	if SecretConfigFileFlag != "" {
+		err = injectSecrets(ctx, k8sC)
+		checkError(err)
+	}
+
 	if !MinimalFlag {
 
 		logrus.Info("Injecting Default Deployments")

cmd/k3auto.go

@@ -14,12 +14,14 @@ var K3AutoCmd = &cobra.Command{
 
 var (
 	ClusterConfigFileFlag   string
+	SecretConfigFileFlag    string
 	DeploymentDirectoryFlag string
 	MinimalFlag             bool
 )
 
 func init() {
 	K3AutoCmd.PersistentFlags().StringVarP(&ClusterConfigFileFlag, "cluster-config", "c", "", "Override Cluster Config File")
+	K3AutoCmd.PersistentFlags().StringVarP(&SecretConfigFileFlag, "secret-config", "s", "", "Inject Secrets To the Cluster on Creation")
 	K3AutoCmd.PersistentFlags().StringVarP(&DeploymentDirectoryFlag, "deployment-directory", "d", "", "Deployment Directory")
 	K3AutoCmd.PersistentFlags().BoolVarP(&MinimalFlag, "minimal", "m", false, "Only deploy the k3d cluster & flux controllers")
 

e2e/e2e_secrets.yaml

@@ -0,0 +1,9 @@
+DefaultSecret: e2e-secret
+DefaultNamespace: default
+Secrets:
+  - Type: exec
+    SecretKey: "alpha"
+    Args:
+      - /bin/bash
+      - -c
+      - "echo -n Hello-World"

e2e/e2e_test.go

@@ -35,7 +35,7 @@ func init() {
 func SetupEnvironment(ctx context.Context) (func(ctx context.Context) error, error) {
 	// go cmd.CreateCmd.ExecuteContext(ctx)
 
-	cmd.K3AutoCmd.SetArgs([]string{"create", "-d", "./e2e_deployments"})
+	cmd.K3AutoCmd.SetArgs([]string{"create", "-d", "./e2e_deployments", "-s", "./e2e_secrets.yaml"})
 	go cmd.K3AutoCmd.ExecuteContext(ctx)
 
 	cleanupFn := func(ctx context.Context) error {

internal/secrets/config.go

@@ -0,0 +1,23 @@
+package secrets
+
+import (
+	"io"
+
+	"gopkg.in/yaml.v3"
+)
+
+func LoadConfigFile(r io.Reader) (SecretConfig, error) {
+	c := NewSecretConfig(10, 10)
+
+	b, err := io.ReadAll(r)
+	if err != nil {
+		return SecretConfig{}, err
+	}
+
+	err = yaml.Unmarshal(b, &c)
+	if err != nil {
+		return SecretConfig{}, err
+	}
+
+	return c, nil
+}

internal/secrets/config.test.yaml

@@ -0,0 +1,18 @@
+DefaultSecret: testsecreta
+DefaultNamespace: default
+Secrets:
+  - Type: exec
+    SecretName: "flux-system"
+    SecretKey: "known-hosts"
+    Args:
+      - /bin/bash
+      - -c
+      - "aws ssm get-parameter --name /lab/flux-known-hosts --with-decryption | jq '.Parameter.Value'"
+
+  - Type: exec
+    SecretName: "flux-system"
+    SecretKey: "flux-public-key"
+    Args:
+      - /bin/bash
+      - -c
+      - "aws ssm get-parameter --name /lab/flux-public-key --with-decryption | jq '.Parameter.Value'"

internal/secrets/config_test.go

@@ -0,0 +1,37 @@
+package secrets
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/pthomison/k3auto/internal/k8s"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfigLoad(t *testing.T) {
+	// return
+	f, err := os.Open("config.test.yaml")
+	assert.Nil(t, err)
+	conf, err := LoadConfigFile(f)
+	assert.Nil(t, err)
+
+	k8sC, err := k8s.NewClient()
+	assert.Nil(t, err)
+
+	err = InjectSecrets(context.TODO(), k8sC, conf)
+	assert.Nil(t, err)
+
+	// spew.Dump(conf)
+}
+
+func TestExec(t *testing.T) {
+	return
+	er := &ExecResolver{}
+
+	val, err := er.Resolve(context.TODO(), []string{"/bin/bash", "-c", "echo -n hello world"})
+	assert.Nil(t, err)
+
+	spew.Dump(val)
+}

internal/secrets/exec.go

@@ -0,0 +1,17 @@
+package secrets
+
+import (
+	"context"
+	"os/exec"
+)
+
+type ExecResolver struct{}
+
+func (r *ExecResolver) Resolve(ctx context.Context, args []string) (string, error) {
+	out, err := exec.CommandContext(ctx, args[0], args[1:]...).Output()
+	if err != nil {
+		return "", err
+	}
+
+	return string(out), nil
+}