diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 84dbe51..d79baa8 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -10,7 +10,7 @@ jobs: E2E: strategy: matrix: - vault: [1.15, 1.16, 1.17] + vault: [1.16, 1.17, 1.18] versions: - k8s_version: v1.28.0 kind_cfg: kind-config_v1.yaml @@ -54,7 +54,7 @@ jobs: - name: setup go uses: actions/setup-go@v5 with: - go-version: '1.22.1' + go-version: '1.22.6' cache: false - name: setup qemu diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7b585f1..1cdc9cc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,8 +24,7 @@ jobs: run: go get ./... - name: Run coverage - run: | - gotestsum -- -v -race -coverprofile="coverage.out" -covermode=atomic ./... + run: make test env: # https://github.com/testcontainers/testcontainers-go/issues/1782 TESTCONTAINERS_RYUK_DISABLED: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a24ec2c..dedddd1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,15 +14,3 @@ repos: args: [--branch, main] - id: pretty-format-json args: [--autofix, --no-sort-keys] - - # - repo: https://github.com/tekwizely/pre-commit-golang - # rev: v1.0.0-rc.1 - # hooks: - # - id: go-test-repo - # - id: go-staticcheck-repo - # - id: go-fmt - # - id: go-fumpt - # - id: go-imports - # - id: go-lint - # - id: golangci-lint-mod - # args: [-c.golang-ci.yml] diff --git a/Makefile b/Makefile index fe0cbb4..55c7895 100644 --- a/Makefile +++ b/Makefile @@ -15,9 +15,8 @@ docs: ## render docs locally mkdocs serve PHONY: test -test: ## display test coverage - go test --cover -parallel=1 -v -coverprofile=coverage.out ./... - go tool cover -func=coverage.out | sort -rnk3 +test: ## test + gotestsum -- -v --shuffle=on -race -coverprofile="coverage.out" -covermode=atomic ./... PHONY: lint lint: ## lint go files @@ -31,10 +30,40 @@ setup-vault: ## setup a local vault dev server with transit engine + key setup-registry: ## setup a local docker registry for pulling in kind ./scripts/local-registry.sh +.PHONY: gen-load +gen-load: ## generate load on KMS plugin + while true; do \ + go run cmd/v2_client/main.go $$(openssl rand -base64 12);\ + done; + +.PHONY: gen-secrets +gen-secrets: ## generate secrets on KMS plugin + while true; do \ + kubectl create secret generic $$(openssl rand -hex 8 | tr '[:upper:]' '[:lower:]')\ + --from-literal=$$(openssl rand -hex 8 | tr '[:upper:]' '[:lower:]')=$$(openssl rand -hex 8 | tr '[:upper:]' '[:lower:]');\ + done; + .PHONY: setup-kind setup-kind: ## setup kind cluster with encrpytion provider configured kind delete cluster --name=kms || true kind create cluster --name=kms --config scripts/kind-config_v2.yaml +.PHONY: setup-o11y +setup-o11y: ## install grafana and prometheus via helm + kubectl apply -f scripts/svc.yml + + helm repo add prometheus-community https://prometheus-community.github.io/helm-charts + helm repo add grafana https://grafana.github.io/helm-charts + helm repo update + + helm install prometheus prometheus-community/prometheus --values scripts/prometheus_values.yml + helm install grafana grafana/grafana --values scripts/grafana_values.yml + + kubectl get secret --namespace default grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo + .PHONY: setup-local setup-local: setup-vault setup-registry setup-kind ## complete local setup + +.PHONY: destroy +destroy: ## destroy kind cluster + kind delete cluster --name=kms diff --git a/README.md b/README.md index c55ed91..4857d80 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Since the key used for encrypting secrets is not stored in Kubernetes, an attack `vault-kubernetes-kms` is supposed to run as a static pod on every control plane node or on that node where the `kube-apiserver` will run. -The plugin creates a Unix-Socket and receive encryption requests through that socket from the `kube-apiserver`. The plugin will then use the specified Vault transit encryption key to encrypt the data and send it back to the `kube-apiserver`, who will then store the encrypted response in `etcd`. +`vault-kubernetes-kms` will start a UNIX domain socket and listens for encryption requests from the `kube-apiserver`. The plugin will then use the specified Vault transit encryption key to encrypt the data and send it back to the `kube-apiserver`, who will then store the encrypted response in `etcd`. To do so, you will have to enable Data at Rest encryption, by configuring the `kube-apiserver` to use a `EncryptionConfiguration` (See [https://falcosuessgott.github.io/vault-kubernetes-kms/configuration/](https://falcosuessgott.github.io/vault-kubernetes-kms/configuration/) for more details). @@ -31,6 +31,7 @@ To do so, you will have to enable Data at Rest encryption, by configuring the `k * support [Vault Token](https://developer.hashicorp.com/vault/docs/auth/token), [AppRole](https://developer.hashicorp.com/vault/docs/auth/approle) authentication (Since a static pod cannot reference any other Kubernetes API-Objects, Vaults Kubernetes Authentication is not possible.) * support Kubernetes [KMS Plugin v1 (deprecated since `v1.28.0`) & v2 (stable in `v1.29.0`)](https://kubernetes.io/docs/tasks/administer-cluster/kms-provider/#before-you-begin) * automatic Token Renewal for avoiding Token expiry +* Exposes useful Prometheus Metrics ## Without a KMS Provider ```bash diff --git a/assets/vault-kubernetes-kms.yml b/assets/vault-kubernetes-kms.yml index c3593ef..9975f9e 100644 --- a/assets/vault-kubernetes-kms.yml +++ b/assets/vault-kubernetes-kms.yml @@ -19,13 +19,21 @@ spec: # mount /opt/kms host directory - name: kms mountPath: /opt/kms + livenessProbe: + httpGet: + path: /health + port: 8080 + readinessProbe: + httpGet: + path: /live + port: 8080 resources: requests: cpu: 100m memory: 128Mi limits: cpu: 2 - memory: 1Gi + memory: 256Mi volumes: # mount /opt/kms host directory - name: kms diff --git a/cmd/plugin.go b/cmd/plugin.go index bb42ae6..d5d5755 100644 --- a/cmd/plugin.go +++ b/cmd/plugin.go @@ -1,21 +1,26 @@ package cmd import ( + "context" "errors" "flag" "fmt" - "log" + "net/http" "os" "os/signal" "slices" "strings" "syscall" + "time" "github.com/FalcoSuessgott/vault-kubernetes-kms/pkg/logging" + "github.com/FalcoSuessgott/vault-kubernetes-kms/pkg/metrics" "github.com/FalcoSuessgott/vault-kubernetes-kms/pkg/plugin" + "github.com/FalcoSuessgott/vault-kubernetes-kms/pkg/probes" "github.com/FalcoSuessgott/vault-kubernetes-kms/pkg/socket" "github.com/FalcoSuessgott/vault-kubernetes-kms/pkg/utils" "github.com/FalcoSuessgott/vault-kubernetes-kms/pkg/vault" + "github.com/prometheus/client_golang/prometheus/promhttp" "go.uber.org/zap" "go.uber.org/zap/zapcore" "google.golang.org/grpc" @@ -42,10 +47,20 @@ type Options struct { AppRoleRoleSecretID string `env:"APPROLE_SECRET_ID"` AppRoleMount string `env:"APPROLE_MOUNT" envDefault:"approle"` + // token refresh + TokenRefreshInterval string `env:"TOKEN_REFRESH_INTERVAL" envDefault:"60s"` + TokenRenewalSeconds int `env:"TOKEN_RENEWAL_SECONDS" envDefault:"3600"` + // transit TransitKey string `env:"TRANSIT_KEY" envDefault:"kms"` TransitMount string `env:"TRANSIT_MOUNT" envDefault:"transit"` + // healthz check + HealthPort string `env:"HEALTH_PORT" envDefault:"8080"` + + // Disable KMSv1 Plugin + DisableV1 bool `env:"DISABLE_V1" envDefault:"true"` + Version bool } @@ -78,9 +93,16 @@ func NewPlugin(version string) error { flag.StringVar(&opts.AppRoleRoleID, "approle-role-id", opts.AppRoleRoleID, "Vault Approle role ID (when approle auth)") flag.StringVar(&opts.AppRoleRoleSecretID, "approle-secret-id", opts.AppRoleRoleSecretID, "Vault Approle Secret ID (when approle auth)") + flag.StringVar(&opts.TokenRefreshInterval, "token-refresh-interval", opts.TokenRefreshInterval, "Interval to check for a token renewal") + flag.IntVar(&opts.TokenRenewalSeconds, "token-renewal", opts.TokenRenewalSeconds, "The number of seconds to renew the token") + flag.StringVar(&opts.TransitMount, "transit-mount", opts.TransitMount, "Vault Transit mount name") flag.StringVar(&opts.TransitKey, "transit-key", opts.TransitKey, "Vault Transit key name") + flag.StringVar(&opts.HealthPort, "health-port", opts.HealthPort, "Health Check Port") + + flag.BoolVar(&opts.DisableV1, "disable-v1", opts.DisableV1, "disable the v1 kms plugin") + flag.BoolVar(&opts.Version, "version", opts.Version, "prints out the plugins version") if err := flag.Parse(os.Args[1:]); err != nil { @@ -90,7 +112,7 @@ func NewPlugin(version string) error { if opts.Version { fmt.Fprintf(os.Stdout, "vault-kubernetes-kms v%s\n", version) - os.Exit(0) + return nil } if err := opts.validateFlags(); err != nil { @@ -111,11 +133,13 @@ func NewPlugin(version string) error { zap.ReplaceGlobals(l) var ( - authMethod vault.Option - logfields []zapcore.Field + authMethod vault.Option + logFields []zapcore.Field + healthChecks = []probes.Prober{} + ctx = shutDownSignal(context.Background()) ) - logfields = append(logfields, + logFields = append(logFields, zap.String("auth-method", opts.AuthMethod), zap.String("socket", opts.Socket), zap.Bool("debug", opts.Debug), @@ -123,26 +147,31 @@ func NewPlugin(version string) error { zap.String("vault-namespace", opts.VaultNamespace), zap.String("transit-engine", opts.TransitMount), zap.String("transit-key", opts.TransitKey), + zap.String("health-port", opts.HealthPort), + zap.String("token-refresh-interval", opts.TokenRefreshInterval), + zap.Int("token-renewal-seconds", opts.TokenRenewalSeconds), + zap.Bool("disable-v1", opts.DisableV1), ) switch strings.ToLower(opts.AuthMethod) { case "token": authMethod = vault.WithTokenAuth(opts.Token) case "approle": - authMethod = vault.WitAppRoleAuth(opts.AppRoleMount, opts.AppRoleRoleID, opts.AppRoleRoleSecretID) - logfields = append(logfields, + authMethod = vault.WithAppRoleAuth(opts.AppRoleMount, opts.AppRoleRoleID, opts.AppRoleRoleSecretID) + logFields = append(logFields, zap.String("approle-mount", opts.AppRoleMount), zap.String("approle-role-id", opts.AppRoleRoleID)) default: return fmt.Errorf("invalid auth method: %s", opts.AuthMethod) } - zap.L().Info("starting kms plugin", logfields...) + zap.L().Info("starting kms plugin", logFields...) vc, err := vault.NewClient( vault.WithVaultAddress(opts.VaultAddress), vault.WithVaultNamespace(opts.VaultNamespace), vault.WithTransit(opts.TransitMount, opts.TransitKey), + vault.WithTokenRenewalSeconds(opts.TokenRenewalSeconds), authMethod, ) if err != nil { @@ -151,6 +180,17 @@ func NewPlugin(version string) error { zap.L().Info("Successfully authenticated to vault") + go func() { + zap.L().Info("Starting token refresher", + zap.String("interval", opts.TokenRefreshInterval), + zap.Int("renewal-seconds", opts.TokenRenewalSeconds), + ) + + t, _ := time.ParseDuration(opts.TokenRefreshInterval) + + vc.LeaseRefresher(ctx, t) + }() + s, err := socket.NewSocket(opts.Socket) if err != nil { zap.L().Fatal("Cannot create socket", zap.Error(err)) @@ -160,19 +200,27 @@ func NewPlugin(version string) error { listener, err := s.Listen(opts.ForceSocketOverwrite) if err != nil { - log.Fatal(fmt.Errorf("failed to listen on socket: %w. Use -force-socket-overwrite (VAULT_KUBERNETES_KMS_FORCE_SOCKET_OVERWRITE)", err)) + zap.L().Fatal("failed to listen on socket: Use -force-socket-overwrite (VAULT_KUBERNETES_KMS_FORCE_SOCKET_OVERWRITE)", + zap.String("socket", opts.Socket), + zap.Any("error", err)) } zap.L().Info("Listening for connection") grpc := grpc.NewServer() - pluginV1 := plugin.NewPluginV1(vc) - pluginV1.Register(grpc) - zap.L().Info("Successfully registered kms plugin v1") + if !opts.DisableV1 { + pluginV1 := plugin.NewPluginV1(vc) + pluginV1.Register(grpc) + + healthChecks = append(healthChecks, pluginV1) + + zap.L().Info("Successfully registered kms plugin v1") + } pluginV2 := plugin.NewPluginV2(vc) pluginV2.Register(grpc) + healthChecks = append(healthChecks, pluginV2) zap.L().Info("Successfully registered kms plugin v2") @@ -182,13 +230,30 @@ func NewPlugin(version string) error { } }() - signals := make(chan os.Signal, 1) - signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) + go func() { + mux := &http.ServeMux{} + + mux.HandleFunc("/metrics", promhttp.HandlerFor(metrics.RegisterPrometheusMetrics(), promhttp.HandlerOpts{}).ServeHTTP) + mux.HandleFunc("/health", probes.HealthZ(healthChecks)) + mux.HandleFunc("/live", probes.HealthZ(healthChecks)) - signal := <-signals + //nolint: mnd + server := &http.Server{ + Addr: ":" + opts.HealthPort, + Handler: mux, + ReadHeaderTimeout: 3 * time.Second, + } + + if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + zap.L().Fatal("Failed to start health check handlers", zap.Error(err)) + } + + zap.L().Info("Exposing metrics under /metrics", zap.String("port", opts.HealthPort)) + zap.L().Info("Exposing health check under /health", zap.String("port", opts.HealthPort)) + zap.L().Info("Exposing live check under /live", zap.String("port", opts.HealthPort)) + }() - zap.L().Info("Received signal", zap.Stringer("signal", signal)) - zap.L().Info("Shutting down server") + <-ctx.Done() grpc.GracefulStop() @@ -215,5 +280,27 @@ func (o *Options) validateFlags() error { return errors.New("approle role id and secret id required when using approle auth") } + if _, err := time.ParseDuration(o.TokenRefreshInterval); err != nil { + return fmt.Errorf("invalid token refresh interval: %w", err) + } + return nil } + +func shutDownSignal(ctx context.Context) context.Context { + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT, os.Interrupt) + + parentCtx, cancel := context.WithCancel(ctx) + + go func() { + signal := <-signalChan + + cancel() + + zap.L().Info("Received signal", zap.Stringer("signal", signal)) + zap.L().Info("Shutting down server") + }() + + return parentCtx +} diff --git a/cmd/plugin_test.go b/cmd/plugin_test.go index e24fa6d..27a62ec 100644 --- a/cmd/plugin_test.go +++ b/cmd/plugin_test.go @@ -33,6 +33,7 @@ func TestNewPlugin(t *testing.T) { "vault-kubernetes-kms", "-auth-method=token", "-token=root", + "-health-port=8081", fmt.Sprintf("-socket=unix:///%s/vaultkms.socket", t.TempDir()), }, extraArgs: func(c *testutils.TestContainer) ([]string, error) { @@ -50,6 +51,7 @@ func TestNewPlugin(t *testing.T) { args: []string{ "vault-kubernetes-kms", "-auth-method=approle", + "-health-port=8082", fmt.Sprintf("-socket=unix:///%s/vaultkms.socket", t.TempDir()), }, extraArgs: func(c *testutils.TestContainer) ([]string, error) { @@ -81,6 +83,7 @@ func TestNewPlugin(t *testing.T) { args: []string{ "vault-kubernetes-kms", "-approle-mount=approle2", + "-health-port=8083", fmt.Sprintf("-socket=unix:///%s/vaultkms.socket", t.TempDir()), }, extraArgs: func(c *testutils.TestContainer) ([]string, error) { @@ -117,9 +120,6 @@ func TestNewPlugin(t *testing.T) { tc.args = append(tc.args, extraArgs...) } - fmt.Println("args", tc.args) - fmt.Println("env", tc.envVars) - os.Args = tc.args var wg sync.WaitGroup diff --git a/cmd/v2_client/README.md b/cmd/v2_client/README.md new file mode 100644 index 0000000..28b908c --- /dev/null +++ b/cmd/v2_client/README.md @@ -0,0 +1,2 @@ +# Usage +See [https://falcosuessgott.github.io/vault-kubernetes-kms/development/](https://falcosuessgott.github.io/vault-kubernetes-kms/development/) diff --git a/cmd/v2_client/main.go b/cmd/v2_client/main.go new file mode 100644 index 0000000..d1b28c5 --- /dev/null +++ b/cmd/v2_client/main.go @@ -0,0 +1,66 @@ +package main + +import ( + "context" + "encoding/base64" + "fmt" + "log" + "os" + "strings" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + pb "k8s.io/kms/apis/v2" +) + +var socket = "/tmp/kms.socket" + +//nolint:funlen,gocritic,cyclop +func main() { + if len(os.Args) == 1 { + fmt.Println("Usage: cmd/v2_Client/main.go ") + + os.Exit(0) + } + + conn, err := grpc.NewClient("unix:"+socket, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("Failed to initialize grpc client: %v", err) + } + + defer conn.Close() + + client := pb.NewKeyManagementServiceClient(conn) + ctx := context.Background() + + _, err = client.Status(ctx, &pb.StatusRequest{}) + if err != nil { + log.Fatalf("Failed to get version: %v", err) + } + + if len(os.Args) > 1 { + input := strings.Join(os.Args[1:], " ") + + encRes, err := client.Encrypt(ctx, &pb.EncryptRequest{Plaintext: []byte(input)}) + if err != nil { + log.Fatalf("Failed to encrypt: %v", err) + } + + resp := base64.StdEncoding.EncodeToString(encRes.GetCiphertext()) + fmt.Printf("\"%s\" -> \"%s\"", input, resp) + + b, err := base64.StdEncoding.DecodeString(resp) + if err != nil { + log.Fatalf("Failed to decode: %v", err) + } + + decResp, err := client.Decrypt(ctx, &pb.DecryptRequest{Ciphertext: b}) + if err != nil { + log.Fatalf("Failed to encrypt: %v", err) + } + + fmt.Printf(" -> \"%s\"\n", string(decResp.GetPlaintext())) + + os.Exit(0) + } +} diff --git a/docs/configuration.md b/docs/configuration.md index c2968f5..175db5a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -15,7 +15,7 @@ The `vault-kms-plugin` requires a Vault Authentication, that allows encrypting a The following steps enable a transit engine `transit` and create transit key `kms`: ```bash $> export VAULT_ADDR="https://vault.tld.de" # change to your Vaults API Address -$> export VAULT_TOKEN="hhvs.XXXX" # change to a token allowed to create a transit engine and a transit key +$> export VAULT_TOKEN="hvs.XXXX" # change to a token allowed to create a transit engine and a transit key $> vault secrets enable transit $> vault write -f transit/keys/kms ``` @@ -113,7 +113,6 @@ List of required and optional CLI args/env vars. **Furthermore, all of Vaults [E * **(Optional)**: `-transit-mount` (`VAULT_KMS_TRANSIT_MOUNT`); default: `"transit"` * **(Optional)**: `-transit-key` (`VAULT_KMS_TRANSIT_KEY`); default: `"kms"` - **If Vault Token Auth**: * **(Required)**: `-auth-method="token"` (`VAULT_KMS_AUTH_METHOD`) @@ -126,17 +125,34 @@ List of required and optional CLI args/env vars. **Furthermore, all of Vaults [E * **(Required)**: `-approle-secret-id` (`VAULT_KMS_APPROLE_SECRET_ID`) * **(Optional)**: `-approle-mount` (`VAULT_KMS_APPROLE_MOUNT`); default: `"approle"` +**Lease Refreshing Settings**: + +* **(Optional)**: `-token-refresh-interval` (`VAULT_KMS_TOKEN_REFRESH_INTERVAL`); default: `"60s"` +* **(Optional)**: `-token-renewal` (`VAULT_KMS_TOKEN_RENEWAL`); default: `"3600"` + +!!! warning + `vault_kubernetes_kms` automatically renewals the lease to avoid expired/revoked leases. + + It does so by periodically comparing the current TTL with the tokens creation TTL (See [https://developer.hashicorp.com/vault/tutorials/get-started/introduction-tokens#token-metadata](https://developer.hashicorp.com/vault/tutorials/get-started/introduction-tokens#token-metadata)). The check will be run periodically in the interval specified with `-token-refresh-interval` (default: `60s`). + + **If the current TTL is less than half of the creation TTL, then the current lease will be renewed for the amount of seconds defined in `-token-renewal` (default: `3600` seconds, `1h`).** + + **If, for whatever reason the token renewal fails, then the configured auth method will be executed again**. Meanwhile this works well with Approle authentication, token auth will most likely fail, due to the token being revoked. In that case, you should make sure to provide a [periodic token](https://falcosuessgott.github.io/vault-kubernetes-kms/configuration/#token-auth). + **General**: * **(Optional)**: `-socket` (`VAULT_KMS_SOCKET`); default: `unix:///opt/kms/vaultkms.socket"` * **(Optional)**: `-force-socket-overwrite` (`FORCE_SOCKET_OVERWRITE`); default: `false`. -!!! note - Use `-force-socket-overwrite` with caution. This will delete whatever filetype exists at the value specified in `-socket`. +!!! warning + Use `-force-socket-overwrite` with caution. **This will delete whatever filetype exists at the value specified in `-socket` path on the Control plane node**. When `vault-kubernetes-kms` crashes, it is not guaranteed that the socket-file will always be removed. For those scenarios `-force-socket-overwrite` was introduced to allow a smooth re-deployment of the plugin and not having to manually delete the stale socket file on the control plane node. * **(Optional)**: `-debug` (`VAULT_KMS_DEBUG`) +* **(Optional)**: `-health-port` (`VAULT_KMS_HEALTH_PORT`); default: `":8080"` +* **(Optional)**: `-disable-v1` (`VAULT_KMS_DISABLE_V1`); default: `"false"` + ### Example Vault Token Auth @@ -155,19 +171,27 @@ spec: # either specify CLI Args or env vars (look above) command: - /vault-kubernetes-kms - - -vault-address=https://vault.server.d + - -vault-address=https://vault.server.de - -auth-method=token - -token=hvs.ABC123 volumeMounts: - name: kms mountPath: /opt/kms + livenessProbe: + httpGet: + path: /health + port: 8080 + readinessProbe: + httpGet: + path: /live + port: 8080 resources: requests: cpu: 100m memory: 128Mi limits: - cpu: "2" - memory: 1Gi + cpu: 2 + memory: 256Mi volumes: - name: kms hostPath: @@ -191,20 +215,28 @@ spec: # either specify CLI Args or env vars (look above) command: - /vault-kubernetes-kms - - -vault-address=https://vault.server.d + - -vault-address=https://vault.server.de - -auth-method=approle - -approle-role-id=XXXX - -approle-secret-id=XXXX volumeMounts: - name: kms mountPath: /opt/kms + livenessProbe: + httpGet: + path: /health + port: 8080 + readinessProbe: + httpGet: + path: /live + port: 8080 resources: requests: cpu: 100m memory: 128Mi limits: - cpu: "2" - memory: 1Gi + cpu: 2 + memory: 256Mi volumes: - name: kms hostPath: @@ -247,13 +279,21 @@ spec: # mount the volume - name: kms mountPath: /opt/kms + livenessProbe: + httpGet: + path: /health + port: 8080 + readinessProbe: + httpGet: + path: /live + port: 8080 resources: requests: cpu: 100m memory: 128Mi limits: - cpu: "2" - memory: 1Gi + cpu: 2 + memory: 256Mi volumes: - name: kms hostPath: diff --git a/docs/dashboard.png b/docs/dashboard.png new file mode 100644 index 0000000..4103b75 Binary files /dev/null and b/docs/dashboard.png differ diff --git a/docs/development.md b/docs/development.md index eb5bf33..32ece71 100644 --- a/docs/development.md +++ b/docs/development.md @@ -1,28 +1,51 @@ # Development +This guide walks you through the required steps of building and running this plugin locally with and without Kubernetes. + +## Local Development without Kubernetes +You don't have to deploy the plugin in a Kubernetes Cluster, you can just execute it locally, given you have a Vault running: + +```bash +$> make setup-vault +$> go run main.go -vault-address=http://127.0.0.1:8200 -auth-method=token -token=root -socket=unix:///tmp/kms.socket +{"level":"info","timestamp":"2024-08-25T15:36:19.233+1000","caller":"cmd/plugin.go:154","message":"starting kms plugin","auth-method":"token","socket":"unix:///tmp/kms.socket","debug":false,"vault-address":"http://127.0.0.1:8200","vault-namespace":"","transit-engine":"transit","transit-key":"kms","health-port":":8080","disable-v1":false} +{"level":"info","timestamp":"2024-08-25T15:36:19.235+1000","caller":"cmd/plugin.go:167","message":"Successfully authenticated to vault"} +{"level":"info","timestamp":"2024-08-25T15:36:19.235+1000","caller":"cmd/plugin.go:174","message":"Successfully dialed to unix domain socket","socket":"unix:///tmp/kms.socket"} +{"level":"info","timestamp":"2024-08-25T15:36:19.235+1000","caller":"cmd/plugin.go:184","message":"Successfully registered kms plugin v1"} +{"level":"info","timestamp":"2024-08-25T15:36:19.235+1000","caller":"cmd/plugin.go:191","message":"Successfully registered kms plugin v2"} +``` + +In order to send encryption and decryption requests you can use the client CLI tool in `cmd/v2_Client/main.go`. This tool simply connects to the plugin and encrypts a given string and decrypts it back to its plaintext version: + +```bash +$> go run cmd/v2_client/main.go encrypt this string +"encrypt this string" -> "dmF1bHQ6djE6VzJMcHp4UmJMdHV4TWNnUnVWMWJQQzBHMWZ0VkwvZFVUMldLRzQ0RUtCa1VJcjVwVjgxMFd3T29pRmVhQzVNPQ==" -> "encrypt this string" +``` + +## Local Development with Kubernetes The following steps describe how to build & run the vault-kubernetes-kms completely locally using `docker`, `vault` & `kind`. -## Requirements -Obviously you will need all the tools mentioned above installed. Also this setup is only tested on Linux & x86 +### Requirements +Obviously you will need all the tools mentioned above installed. Also this setup is only tested on Linux and MacOS. -## Components +### Components Basically, we will need: 1. A local Vault server initialized & unsealed and with a transit engine enabled as well as a transit key created. 2. A local (docker) registry so kind can pull the currently unreleased `vault-kubernetes-kms` image. 3. A local Kubernetes Cluster (kind) configured to use the local registry as well as the required settings for the kube-apiservers encryption provider config. -### 1. Local Vault Server using `vault` -The following snippets sets up a local vault development server and creates a transit engine as well as a key. +#### 1. Local Vault Server using `vault` +The following snippets sets up a local vault development server and creates a transit engine as well as a transit key. -This script is located in `scripts/vault.sh` and available via `make setup-vault`: +This script is located in `scripts/vault.sh` and is available via `make setup-vault`: ```bash {!../scripts/vault.sh!} ``` -### 2. Local Container/Docker Registry using `docker` +#### 2. Local Container/Docker Registry using `docker` The following snippet, starts a local container registry, builds the current commits `vault-kubernetes-kms` image and tags & pushes the image to the local registry. This script is located in `scripts/local-registry.sh` and is available via `make setup-registry`: @@ -31,7 +54,7 @@ This script is located in `scripts/local-registry.sh` and is available via `make {!../scripts/local-registry.sh!} ``` -### 3. Local Kubernetes Cluster using `kind` +#### 3. Local Kubernetes Cluster using `kind` Last but not least, we combine the above mentioned tools and consume them with `kind` The following `kind`-config configures the local running registry, copies the encryption provider config and the `vault-kubernetes-kms` static pod manifest to the Kubernetes host and patches the `kube-apiserver` for using the provided encryption provider config. @@ -42,8 +65,9 @@ This can be run via `make setup-kind`, which runs `kind create cluster --name=km {!../scripts/kind-config_v2.yaml!} ``` -#### the `vault-kubernetes-kms` manifest -for development purposes, we use the vault dev servers configured root token (`"root"`) as well as the docker ip of the localhost, where the vault server is running (`172.18.0.1`): +**the `vault-kubernetes-kms` manifest:** + +for development purposes, we use the vault dev servers configured root token (`"root"`): ```yaml {!../scripts/vault-kubernetes-kms.yml!} diff --git a/docs/index.md b/docs/index.md index f042dba..a0e4bc3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,3 +29,4 @@ To do so, you will have to enable Data at Rest encryption, by configuring the `k * support [Vault Token Auth](https://developer.hashicorp.com/vault/docs/auth/token) (not recommended for production), [AppRole](https://developer.hashicorp.com/vault/docs/auth/approle) and [Vault Kubernetes Auth](https://developer.hashicorp.com/vault/docs/auth/kubernetes) using the Plugins Service Account * support Kubernetes [KMS Plugin v1 (deprecated since `v1.28.0`) & v2 (stable in `v1.29.0`)](https://kubernetes.io/docs/tasks/administer-cluster/kms-provider/#before-you-begin) * automatic Token Renewal for avoiding Token expiry +* Exposes useful Prometheus Metrics diff --git a/docs/metrics.md b/docs/metrics.md new file mode 100644 index 0000000..1d09b1c --- /dev/null +++ b/docs/metrics.md @@ -0,0 +1,20 @@ +# Prometheus Metrics +Beginning with `v1.0.0` `vault-kubernetes-kms` exposes metrics under `:8080/metrics` (change with `-health-port` or setting `HEALTH_PORT`). + +The following metrics are available: + +## Available Prometheus Metrics +| Metric Name | Type | Description | +|---------------------------------------------------------------------|-----------|-----------------------------------------------------| +| `vault_kubernetes_kms_decryption_operation_duration_seconds_bucket` | Histogram | duration of decryption operations in seconds | +| `vault_kubernetes_kms_encryption_operation_duration_seconds_bucket` | Histogram | duration of encryption operations in seconds | +| `vault_kubernetes_kms_decryption_operation_errors_total` | Counter | total number of errors during decryption operations | +| `vault_kubernetes_kms_encryption_operation_errors_total` | Counter | total number of errors during encryption operations | +| `vault_kubernetes_kms_token_expiry_seconds` | Gauge | time remaining until the current token expires | +| `vault_kubernetes_kms_token_renewals_total` | Counter | total number of token renewals | + +Including the metrics defined in the [Prometheus Process Collector](https://github.com/prometheus/client_golang/blob/main/prometheus/process_collector.go#L38) (when running on `Linux`). + +Those metrics allow you to define your own Grafana Dashboard: + +![img](./dashboard.png) diff --git a/go.mod b/go.mod index dfed03d..aa2a4da 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,11 @@ module github.com/FalcoSuessgott/vault-kubernetes-kms -go 1.22.0 - -toolchain go1.22.6 +go 1.22.6 require ( github.com/caarlos0/env/v6 v6.10.1 - github.com/hashicorp/vault/api v1.15.0 + github.com/hashicorp/vault/api v1.14.0 + github.com/prometheus/client_golang v1.20.1 github.com/stretchr/testify v1.9.0 github.com/testcontainers/testcontainers-go v0.34.0 github.com/testcontainers/testcontainers-go/modules/vault v0.34.0 @@ -21,8 +20,11 @@ require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/bitfield/gotestdox v0.2.2 // indirect - github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cenkalti/backoff/v3 v3.2.2 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect @@ -52,7 +54,7 @@ require ( github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect github.com/hashicorp/go-sockaddr v1.0.6 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/klauspost/compress v1.17.5 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -66,11 +68,15 @@ require ( github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.0 // indirect github.com/morikuni/aec v1.0.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/shirou/gopsutil/v3 v3.23.12 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect diff --git a/go.sum b/go.sum index e401034..20321a4 100644 --- a/go.sum +++ b/go.sum @@ -6,12 +6,21 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bitfield/gotestdox v0.2.2 h1:x6RcPAbBbErKLnapz1QeAlf3ospg8efBsedU93CDsnE= github.com/bitfield/gotestdox v0.2.2/go.mod h1:D+gwtS0urjBrzguAkTM2wodsTQYFHdpx8eqRJ3N+9pY= github.com/caarlos0/env/v6 v6.10.1 h1:t1mPSxNpei6M5yAeu1qtRdPAK29Nbcf/n3G7x+b3/II= github.com/caarlos0/env/v6 v6.10.1/go.mod h1:hvp/ryKXKipEkcuYjs9mI4bBCg+UI0Yhgm5Zu0ddvwc= +github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M= +github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/containerd/containerd v1.7.18/go.mod h1:IYEk9/IO6wAPUz2bCMVUbsfXjzw5UNP5fLz4PsUygQ4= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= @@ -88,16 +97,20 @@ github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31 github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= github.com/hashicorp/vault-client-go v0.4.3 h1:zG7STGVgn/VK6rnZc0k8PGbfv2x/sJExRKHSUg3ljWc= github.com/hashicorp/vault-client-go v0.4.3/go.mod h1:4tDw7Uhq5XOxS1fO+oMtotHL7j4sB9cp0T7U6m4FzDY= +github.com/hashicorp/vault/api v1.14.0 h1:Ah3CFLixD5jmjusOgm8grfN9M0d+Y8fVR2SW0K6pJLU= +github.com/hashicorp/vault/api v1.14.0/go.mod h1:pV9YLxBGSz+cItFDd8Ii4G17waWOQ32zVjMWHe/cOqk= github.com/hashicorp/vault/api v1.15.0 h1:O24FYQCWwhwKnF7CuSqP30S51rTV7vz1iACXE/pj5DA= github.com/hashicorp/vault/api v1.15.0/go.mod h1:+5YTO09JGn0u+b6ySD/LLVf8WkJCPLAL2Vkmrn2+CM8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.5 h1:d4vBd+7CHydUqpFBgUEKkSdtSugf9YFmSkvUYPquI5E= -github.com/klauspost/compress v1.17.5/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed h1:036IscGBfJsFIgJQzlui7nK1Ncm0tp2ktmPj8xO4N/0= github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= @@ -128,6 +141,8 @@ github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= @@ -140,6 +155,14 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig= github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_golang v1.20.1 h1:IMJXHOD6eARkQpxo8KkhgEVFlBNm+nkrFUyGlIu7Na8= +github.com/prometheus/client_golang v1.20.1/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= diff --git a/mkdocs.yml b/mkdocs.yml index 419f2fc..0c5560c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -53,6 +53,7 @@ nav: - configuration.md - concepts.md - sign.md + - metrics.md - integration.md - troubleshooting.md - development.md diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 0000000..a8e9bfc --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,80 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" +) + +const MetricsPrefix = "vault_kubernetes_kms" + +// nolint: mnd +var defaultBuckets = prometheus.ExponentialBuckets(0.001, 2, 11) + +var metricsPrefix = func(s string) string { + return MetricsPrefix + "_" + s +} + +func RegisterPrometheusMetrics() *prometheus.Registry { + promReg := prometheus.NewRegistry() + + promReg.MustRegister( + // note: The process collector only collects metrics on Linux OS. + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{ + Namespace: MetricsPrefix, + }), + EncryptionErrorsTotal, + DecryptionErrorsTotal, + EncryptionOperationDurationSeconds, + DecryptionOperationDurationSeconds, + VaultTokenRenewalTotal, + VaultTokenExpirySeconds, + ) + + return promReg +} + +var ( + EncryptionOperationDurationSeconds = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Name: metricsPrefix("encryption_operation_duration_seconds"), + Help: "duration of encryption operations", + Buckets: defaultBuckets, + }, + ) + + DecryptionOperationDurationSeconds = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Name: metricsPrefix("decryption_operation_duration_seconds"), + Help: "duration of decryption operations", + Buckets: defaultBuckets, + }, + ) + + EncryptionErrorsTotal = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: metricsPrefix("encryption_operation_errors_total"), + Help: "total number of errors during encryption operations", + }, + ) + + DecryptionErrorsTotal = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: metricsPrefix("decryption_operation_errors_total"), + Help: "total number of errors during decryption operations", + }, + ) + + VaultTokenRenewalTotal = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: metricsPrefix("token_renewals_total"), + Help: "total number of token renewals", + }, + ) + + VaultTokenExpirySeconds = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: metricsPrefix("token_expiry_seconds"), + Help: "time remaining until the current token expires", + }, + ) +) diff --git a/pkg/plugin/plugin_test.go b/pkg/plugin/plugin_test.go index 6426d2b..d584d51 100644 --- a/pkg/plugin/plugin_test.go +++ b/pkg/plugin/plugin_test.go @@ -27,7 +27,7 @@ type PluginSuite struct { func TestVaultSuite(t *testing.T) { // github actions doesn't offer the docker sock, which we require for testing - if runtime.GOOS == "linux" { + if runtime.GOOS != "windows" { suite.Run(t, new(PluginSuite)) } } diff --git a/pkg/plugin/plugin_v1.go b/pkg/plugin/plugin_v1.go index 53ec1ce..232e2a5 100644 --- a/pkg/plugin/plugin_v1.go +++ b/pkg/plugin/plugin_v1.go @@ -2,8 +2,11 @@ package plugin import ( "context" + "errors" + "github.com/FalcoSuessgott/vault-kubernetes-kms/pkg/metrics" "github.com/FalcoSuessgott/vault-kubernetes-kms/pkg/vault" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" "google.golang.org/grpc" pb "k8s.io/kms/apis/v1beta1" @@ -11,16 +14,12 @@ import ( // PluginV1 a kms plugin wrapper. type PluginV1 struct { - vc *vault.Client + *vault.Client } // NewPluginV1 returns a kms wrapper. func NewPluginV1(vc *vault.Client) *PluginV1 { - p := &PluginV1{ - vc: vc, - } - - return p + return &PluginV1{vc} } // nolint: staticcheck @@ -32,14 +31,48 @@ func (p *PluginV1) Version(ctx context.Context, request *pb.VersionRequest) (*pb }, nil } +// Health sends a simple plaintext for encryption and then compares the decrypted value. +// nolint: staticcheck +func (p *PluginV1) Health() error { + health := "health" + + enc, err := p.Encrypt(context.Background(), &pb.EncryptRequest{ + Plain: []byte(health), + }) + if err != nil { + return err + } + + dec, err := p.Decrypt(context.Background(), &pb.DecryptRequest{ + Cipher: enc.GetCipher(), + }) + if err != nil { + return err + } + + if health != string(dec.GetPlain()) { + zap.L().Info("v1 health status failed") + + return errors.New("v1 health check failed") + } + + return nil +} + // nolint: staticcheck func (p *PluginV1) Encrypt(ctx context.Context, request *pb.EncryptRequest) (*pb.EncryptResponse, error) { - resp, _, err := p.vc.Encrypt(ctx, request.GetPlain()) + timer := prometheus.NewTimer(metrics.EncryptionOperationDurationSeconds) + + resp, _, err := p.Client.Encrypt(ctx, request.GetPlain()) if err != nil { + metrics.EncryptionErrorsTotal.Inc() + return nil, err } - zap.L().Info("encryption request") + zap.L().Info("v1 encryption request") + + timer.ObserveDuration() return &pb.EncryptResponse{ Cipher: resp, @@ -48,12 +81,18 @@ func (p *PluginV1) Encrypt(ctx context.Context, request *pb.EncryptRequest) (*pb // nolint: staticcheck func (p *PluginV1) Decrypt(ctx context.Context, request *pb.DecryptRequest) (*pb.DecryptResponse, error) { - resp, err := p.vc.Decrypt(ctx, request.GetCipher()) + timer := prometheus.NewTimer(metrics.DecryptionOperationDurationSeconds) + + resp, err := p.Client.Decrypt(ctx, request.GetCipher()) if err != nil { + metrics.DecryptionErrorsTotal.Inc() + return nil, err } - zap.L().Info("decryption request") + zap.L().Info("v1 decryption request") + + timer.ObserveDuration() return &pb.DecryptResponse{ Plain: resp, diff --git a/pkg/plugin/plugin_v2.go b/pkg/plugin/plugin_v2.go index 5c84f1c..ef88165 100644 --- a/pkg/plugin/plugin_v2.go +++ b/pkg/plugin/plugin_v2.go @@ -6,7 +6,9 @@ import ( "strconv" "time" + "github.com/FalcoSuessgott/vault-kubernetes-kms/pkg/metrics" "github.com/FalcoSuessgott/vault-kubernetes-kms/pkg/vault" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" "google.golang.org/grpc" pb "k8s.io/kms/apis/v2" @@ -14,16 +16,42 @@ import ( // PluginV2 a kms plugin wrapper. type PluginV2 struct { - vc *vault.Client + *vault.Client } // PluginV2 returns a kms wrapper. func NewPluginV2(vc *vault.Client) *PluginV2 { - p := &PluginV2{ - vc: vc, + return &PluginV2{vc} +} + +// Status performs a simple health check and returns ok if encryption / decryption was successful +// https://kubernetes.io/docs/tasks/administer-cluster/kms-provider/#developing-a-kms-plugin-gRPC-server-notes-kms-v2 +func (p *PluginV2) Status(ctx context.Context, _ *pb.StatusRequest) (*pb.StatusResponse, error) { + health := "ok" + + kv, err := p.Client.GetKeyVersion(ctx) + if err != nil { + return nil, err } - return p + //nolint: contextcheck + if err := p.Health(); err != nil { + health = "err" + + zap.L().Info(err.Error()) + } + + zap.L().Info("health status", + zap.String("key_id", kv), + zap.String("healthz", health), + zap.String("version", "v2"), + ) + + return &pb.StatusResponse{ + Version: "v2", + Healthz: "ok", + KeyId: kv, + }, nil } // Health sends a simple plaintext for encryption and then compares the decrypted value. @@ -49,58 +77,28 @@ func (p *PluginV2) Health() error { } if health != string(dec.GetPlaintext()) { - zap.L().Info("Health status failed") + zap.L().Info("v2 health status failed") - return errors.New("health check failed") + return errors.New("v2 health check failed") } return nil } -// Status performs a simple health check and returns ok if encryption / decryption was successful -// https://kubernetes.io/docs/tasks/administer-cluster/kms-provider/#developing-a-kms-plugin-gRPC-server-notes-kms-v2 -func (p *PluginV2) Status(ctx context.Context, _ *pb.StatusRequest) (*pb.StatusResponse, error) { - health := "ok" - - if err := p.vc.TokenRefresh(); err != nil { - health = "err" - - zap.L().Info(err.Error()) - } +func (p *PluginV2) Encrypt(ctx context.Context, request *pb.EncryptRequest) (*pb.EncryptResponse, error) { + timer := prometheus.NewTimer(metrics.EncryptionOperationDurationSeconds) - kv, err := p.vc.GetKeyVersion(ctx) + resp, id, err := p.Client.Encrypt(ctx, request.GetPlaintext()) if err != nil { - return nil, err - } - - //nolint: contextcheck - if err := p.Health(); err != nil { - health = "err" - - zap.L().Info(err.Error()) - } - - zap.L().Info("health status", - zap.String("key_id", kv), - zap.String("healthz", health), - zap.String("version", "v2"), - ) - - return &pb.StatusResponse{ - Version: "v2", - Healthz: "ok", - KeyId: kv, - }, nil -} + metrics.EncryptionErrorsTotal.Inc() -func (p *PluginV2) Encrypt(ctx context.Context, request *pb.EncryptRequest) (*pb.EncryptResponse, error) { - resp, id, err := p.vc.Encrypt(ctx, request.GetPlaintext()) - if err != nil { return nil, err } zap.L().Info("v2 encryption request", zap.String("request_id", request.GetUid())) + timer.ObserveDuration() + return &pb.EncryptResponse{ Ciphertext: resp, KeyId: id, @@ -108,13 +106,19 @@ func (p *PluginV2) Encrypt(ctx context.Context, request *pb.EncryptRequest) (*pb } func (p *PluginV2) Decrypt(ctx context.Context, request *pb.DecryptRequest) (*pb.DecryptResponse, error) { - resp, err := p.vc.Decrypt(ctx, request.GetCiphertext()) + timer := prometheus.NewTimer(metrics.DecryptionOperationDurationSeconds) + + resp, err := p.Client.Decrypt(ctx, request.GetCiphertext()) if err != nil { + metrics.DecryptionErrorsTotal.Inc() + return nil, err } zap.L().Info("v2 decryption request", zap.String("request_id", request.GetUid())) + timer.ObserveDuration() + return &pb.DecryptResponse{ Plaintext: resp, }, nil diff --git a/pkg/probes/probes.go b/pkg/probes/probes.go new file mode 100644 index 0000000..e28fda7 --- /dev/null +++ b/pkg/probes/probes.go @@ -0,0 +1,38 @@ +package probes + +import ( + "fmt" + "net/http" + + "go.uber.org/zap" +) + +// Prober interface. +type Prober interface { + Health() error +} + +// HealthZ performs a health check for each prober and returns OK if all checks were successful. +func HealthZ(prober []Prober) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + for _, p := range prober { + if p == nil { + return + } + + if err := p.Health(); err != nil { + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprint(w, err) + + zap.L().Error("health check failed", zap.Error(err)) + + return + } + } + + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, http.StatusText(http.StatusOK)) + + zap.L().Debug("health checks succeeded") + } +} diff --git a/pkg/probes/probes_test.go b/pkg/probes/probes_test.go new file mode 100644 index 0000000..8f19fd3 --- /dev/null +++ b/pkg/probes/probes_test.go @@ -0,0 +1,46 @@ +package probes + +import ( + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/require" +) + +type ErrorProber struct{} + +func (p *ErrorProber) Health() error { return errors.New("probe failed") } + +type SuccessProber struct{} + +func (p *SuccessProber) Health() error { return nil } + +func TestHealthZ(t *testing.T) { + t.Run("error", func(t *testing.T) { + prober := []Prober{&ErrorProber{}} + + hf := HealthZ(prober) + + req := httptest.NewRequest(http.MethodGet, "https://google.de", nil) + w := httptest.NewRecorder() + hf(w, req) + + require.Equal(t, http.StatusInternalServerError, w.Code) + require.Equal(t, "probe failed", w.Body.String()) + }) + + t.Run("succcess", func(t *testing.T) { + prober := []Prober{&SuccessProber{}} + + hf := HealthZ(prober) + + req := httptest.NewRequest(http.MethodGet, "https://google.de", nil) + w := httptest.NewRecorder() + hf(w, req) + + require.Equal(t, http.StatusOK, w.Code) + require.Equal(t, http.StatusText(http.StatusOK), w.Body.String()) + }) +} diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go index d217e7e..2d569c7 100644 --- a/pkg/testutils/testutils.go +++ b/pkg/testutils/testutils.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "io" + "log" + "strings" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/modules/vault" @@ -72,19 +74,27 @@ func (v *TestContainer) GetApproleCreds(mount, role string) (string, string, err } // nolint: perfsprint -func (v *TestContainer) GetToken(policy string) (string, error) { - _, r, err := v.Container.Exec(context.Background(), []string{"vault", "token", "create", "-field=token", fmt.Sprintf("-policy=%s", policy)}) +func (v *TestContainer) GetToken(policy string, ttl string) (string, error) { + return v.RunCommand("vault token create -field=token -policy=" + policy + " -ttl=" + ttl) +} + +func (v *TestContainer) RunCommand(cmd string) (string, error) { + log.Println("running command: ", cmd) + + _, r, err := v.Container.Exec(context.Background(), strings.Split(cmd, " ")) if err != nil { - return "", fmt.Errorf("error creating role_id: %w", err) + return "", fmt.Errorf("error creating root token: %w", err) } - token, err := io.ReadAll(r) + rootToken, err := io.ReadAll(r) if err != nil { - return "", fmt.Errorf("error reading role_id: %w", err) + return "", fmt.Errorf("error reading root token: %w", err) } + log.Println("output: ", string(rootToken[8:])) + // removing the first 8 bytes, which is the shell prompt - return string(token[8:]), nil + return string(rootToken[8:]), nil } // Terminate terminates the testcontainer. diff --git a/pkg/testutils/testutils_test.go b/pkg/testutils/testutils_test.go index 14a81cb..0c0d3ce 100644 --- a/pkg/testutils/testutils_test.go +++ b/pkg/testutils/testutils_test.go @@ -17,7 +17,7 @@ func TestVaultConnection(t *testing.T) { require.NoError(t, err, "start") // create token - token, err := tc.GetToken("default") + token, err := tc.GetToken("default", "1h") require.NoError(t, err, "token creation") tokenVault, err := vault.NewClient( @@ -40,7 +40,7 @@ func TestVaultConnection(t *testing.T) { _, err = vault.NewClient( vault.WithVaultAddress(tc.URI), vault.WithTokenAuth(tc.Token), - vault.WitAppRoleAuth("approle", roleID, secretID), + vault.WithAppRoleAuth("approle", roleID, secretID), vault.WithTransit("transit", "kms"), ) diff --git a/pkg/vault/client.go b/pkg/vault/client.go index 40238db..85fee3d 100644 --- a/pkg/vault/client.go +++ b/pkg/vault/client.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/hashicorp/vault/api" - "go.uber.org/zap" ) // Client Vault API wrapper. @@ -17,6 +16,10 @@ type Client struct { AppRoleID string AppRoleSecretID string + AuthMethodFunc Option + + TokenRenewalSeconds int + TransitEngine string TransitKey string } @@ -26,15 +29,15 @@ type Option func(*Client) error // NewClient returns a new vault client wrapper. func NewClient(opts ...Option) (*Client, error) { + cfg := api.DefaultConfig() + // read all vault env vars - c, err := api.NewClient(api.DefaultConfig()) + c, err := api.NewClient(cfg) if err != nil { return nil, err } - client := &Client{ - Client: c, - } + client := &Client{Client: c} for _, opt := range opts { if err := opt(client); err != nil { @@ -88,12 +91,25 @@ func WithTokenAuth(token string) Option { c.SetToken(token) } + if c.AuthMethodFunc == nil { + c.AuthMethodFunc = WithTokenAuth(token) + } + + return nil + } +} + +// WithTokenAuth sets the specified token. +func WithTokenRenewalSeconds(seconds int) Option { + return func(c *Client) error { + c.TokenRenewalSeconds = seconds + return nil } } // WitAppRoleAuth performs a approle auth login. -func WitAppRoleAuth(mount, roleID, secretID string) Option { +func WithAppRoleAuth(mount, roleID, secretID string) Option { return func(c *Client) error { c.AppRoleID = roleID c.AppRoleMount = mount @@ -111,20 +127,10 @@ func WitAppRoleAuth(mount, roleID, secretID string) Option { c.SetToken(s.Auth.ClientToken) - return nil - } -} + if c.AuthMethodFunc == nil { + c.AuthMethodFunc = WithAppRoleAuth(mount, roleID, secretID) + } -// TokenRefresh renews the token for 24h. -func (c *Client) TokenRefresh() error { - token, err := c.Auth().Token().RenewSelf(tokenRefreshIntervall) - if err != nil { - return fmt.Errorf("error renewing token: %w", err) + return nil } - - c.SetToken(token.Auth.ClientToken) - - zap.L().Info("successfully refreshed token") - - return nil } diff --git a/pkg/vault/client_test.go b/pkg/vault/client_test.go index b4f05f3..bb4680b 100644 --- a/pkg/vault/client_test.go +++ b/pkg/vault/client_test.go @@ -9,7 +9,6 @@ import ( "github.com/FalcoSuessgott/vault-kubernetes-kms/pkg/testutils" "github.com/stretchr/testify/suite" - "github.com/testcontainers/testcontainers-go/exec" ) type VaultSuite struct { @@ -51,11 +50,10 @@ func (s *VaultSuite) SetupSubTest() { // nolint: funlen func (s *VaultSuite) TestAuthMethods() { testCases := []struct { - name string - prepCmd []string - cmdOptions []exec.ProcessOption - auth func() (Option, error) - err bool + name string + prepCmd []string + auth func() (Option, error) + err bool }{ { name: "basic approle auth", @@ -69,20 +67,20 @@ func (s *VaultSuite) TestAuthMethods() { return nil, err } - return WitAppRoleAuth("approle", roleID, secretID), nil + return WithAppRoleAuth("approle", roleID, secretID), nil }, }, { name: "invalid approle auth", err: true, auth: func() (Option, error) { - return WitAppRoleAuth("approle", "invalid", "invalid"), nil + return WithAppRoleAuth("approle", "invalid", "invalid"), nil }, }, { name: "token auth", auth: func() (Option, error) { - token, err := s.tc.GetToken("default") + token, err := s.tc.GetToken("default", "1h") if err != nil { return nil, err } @@ -125,7 +123,7 @@ func (s *VaultSuite) TestAuthMethods() { func TestVaultSuite(t *testing.T) { // github actions doesn't offer the docker sock, which we require for testing - if runtime.GOOS == "linux" { + if runtime.GOOS != "windows" { suite.Run(t, new(VaultSuite)) } } diff --git a/pkg/vault/const.go b/pkg/vault/const.go index 7757ed2..38116e2 100644 --- a/pkg/vault/const.go +++ b/pkg/vault/const.go @@ -8,6 +8,4 @@ const ( mountEnginePath = "sys/mounts/%s" transitKeyPath = "%s/keys/%s" - - tokenRefreshIntervall = 3600 ) diff --git a/pkg/vault/lease.go b/pkg/vault/lease.go new file mode 100644 index 0000000..a3c784e --- /dev/null +++ b/pkg/vault/lease.go @@ -0,0 +1,89 @@ +package vault + +import ( + "context" + "encoding/json" + "time" + + "github.com/FalcoSuessgott/vault-kubernetes-kms/pkg/metrics" + "go.uber.org/zap" +) + +// LeaseRefresher periodically checks the ttl of the current lease and attempts to renew it if the ttl is less than half of the creation ttl. +// if the token renewal fails, a new login with the configured auth method is performed +// this func is supposed to run as a goroutine. +// nolint: funlen, gocognit, cyclop +func (c *Client) LeaseRefresher(ctx context.Context, interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + token, err := c.Auth().Token().LookupSelf() + if err != nil { + zap.L().Error("failed to lookup token", zap.Error(err)) + + continue + } + + creationTTL, ok := token.Data["creation_ttl"].(json.Number) + if !ok { + zap.L().Error("failed to assert creation_ttl type") + + continue + } + + ttl, ok := token.Data["ttl"].(json.Number) + if !ok { + zap.L().Error("failed to assert ttl type") + + continue + } + + creationTTLFloat, err := creationTTL.Float64() + if err != nil { + zap.L().Error("failed to parse creation_ttl", zap.Error(err)) + + continue + } + + ttlFloat, err := ttl.Float64() + if err != nil { + zap.L().Error("failed to parse ttl", zap.Error(err)) + + continue + } + + metrics.VaultTokenExpirySeconds.Set(ttlFloat) + + zap.L().Info("checking token renewal", zap.Float64("creation_ttl", creationTTLFloat), zap.Float64("ttl", ttlFloat)) + + //nolint: nestif + if ttlFloat < creationTTLFloat/2 { + zap.L().Info("attempting token renewal", zap.Int("renewal_seconds", c.TokenRenewalSeconds)) + + if _, err := c.Auth().Token().RenewSelf(c.TokenRenewalSeconds); err != nil { + zap.L().Error("failed to renew token, performing new authentication", zap.Error(err)) + + if err := c.AuthMethodFunc(c); err != nil { + zap.L().Error("failed to authenticate", zap.Error(err)) + } else { + zap.L().Info("successfully re-authenticated") + } + } else { + zap.L().Info("successfully refreshed token") + } + + metrics.VaultTokenRenewalTotal.Inc() + } else { + zap.L().Info("skipping token renewal") + } + + case <-ctx.Done(): + zap.L().Info("token refresher shutting down") + + return + } + } +} diff --git a/pkg/vault/lease_test.go b/pkg/vault/lease_test.go new file mode 100644 index 0000000..ddfe073 --- /dev/null +++ b/pkg/vault/lease_test.go @@ -0,0 +1,32 @@ +package vault + +import ( + "context" + "time" +) + +func (s *VaultSuite) TestTokenRefresher() { + // here we simply create a token with a TTL of 7sec and start the token refresher + // after 20sec we check if the token is still valid + s.Run("token refresher", func() { + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) + defer cancel() + + token, err := s.tc.GetToken("default", "7s") + s.Require().NoError(err, "token creation failed") + + vc, err := NewClient( + WithVaultAddress(s.tc.URI), + WithTokenAuth(token), + ) + + s.Require().NoError(err, "client") + + go vc.LeaseRefresher(ctx, 3*time.Second) + + time.Sleep(15 * time.Second) + + _, err = s.tc.RunCommand("vault token lookup " + token) + s.Require().NoError(err, "token lookup") + }) +} diff --git a/scripts/grafana_values.yml b/scripts/grafana_values.yml new file mode 100644 index 0000000..2bf4fed --- /dev/null +++ b/scripts/grafana_values.yml @@ -0,0 +1,8 @@ +# https://github.com/grafana/helm-charts/blob/main/charts/grafana/values.yaml +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + url: http://prometheus-server:80 diff --git a/scripts/local-registry.sh b/scripts/local-registry.sh index b601c67..87f4088 100755 --- a/scripts/local-registry.sh +++ b/scripts/local-registry.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash +set -xeu -set -eu +command -v docker >/dev/null 2>&1 || { echo "docker is not installed. Aborting." >&2; exit 1; } REGISTRY_NAME=registry REGISTRY_PORT=5000 diff --git a/scripts/prometheus_values.yml b/scripts/prometheus_values.yml new file mode 100644 index 0000000..da0d6d2 --- /dev/null +++ b/scripts/prometheus_values.yml @@ -0,0 +1,18 @@ +# https://github.com/prometheus-community/helm-charts/blob/main/charts/prometheus/values.yaml +alertmanager: + enabled: false + +kube-state-metrics: + enabled: false + +prometheus-node-exporter: + enabled: false + +prometheus-pushgateway: + enabled: false + +extraScrapeConfigs: | + - job_name: vault-kubernetes-kms + metrics_path: /metrics + static_configs: + - targets: ["vault-kubernetes-kms.kube-system.svc.cluster.local:80"] diff --git a/scripts/svc.yml b/scripts/svc.yml new file mode 100644 index 0000000..393c82e --- /dev/null +++ b/scripts/svc.yml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: vault-kubernetes-kms + namespace: kube-system +spec: + selector: + app: vault-kubernetes-kms + ports: + - protocol: TCP + port: 80 + targetPort: 8080 diff --git a/scripts/vault-kubernetes-kms.yml b/scripts/vault-kubernetes-kms.yml index 0f31e4f..20397a2 100644 --- a/scripts/vault-kubernetes-kms.yml +++ b/scripts/vault-kubernetes-kms.yml @@ -3,6 +3,8 @@ kind: Pod metadata: name: vault-kubernetes-kms namespace: kube-system + labels: + app: vault-kubernetes-kms spec: priorityClassName: system-node-critical hostNetwork: true @@ -12,20 +14,28 @@ spec: imagePullPolicy: IfNotPresent command: - /vault-kubernetes-kms - - -vault-address=http://172.17.0.1:8200 + - -vault-address=http://host.docker.internal:8200 - -auth-method=token - -token=root volumeMounts: # mount /opt/kms host directory - name: kms mountPath: /opt/kms + livenessProbe: + httpGet: + path: /health + port: 8080 + readinessProbe: + httpGet: + path: /live + port: 8080 resources: requests: cpu: 100m memory: 128Mi limits: - cpu: "2" - memory: 1Gi + cpu: 2 + memory: 256Mi volumes: # mount /opt/kms host directory - name: kms