tmp

FalcoSuessgott · Mar 26, 2024 · 9b0e3b0 · 9b0e3b0
1 parent 8e07ea6
commit 9b0e3b0
Show file tree

Hide file tree

Showing 18 changed files with 456 additions and 385 deletions.
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -0,0 +1,64 @@
+name: E2E KMSv2
+
+on:
+  pull_request:
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+
+    services:
+      vault:
+        image: hashicorp/vault:1.15
+        env:
+          VAULT_DEV_ROOT_TOKEN_ID: root
+        credentials:
+            username: ${{ secrets.DOCKERHUB_USERNAME }}
+            password: ${{ secrets.DOCKERHUB_TOKEN }}  
+        ports:
+          - 8200:8200
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
+
+    steps:
+      - name: connect registry to kind
+        run: |
+          docker network create kind
+          docker network connect kind registry
+
+      - name: configure vault
+        run: |
+            curl -X POST -H "X-Vault-Token: root" -d '{"type":"transit"}' http://localhost:8200/v1/sys/mounts/transit
+            curl -X PUT -H "X-Vault-Token: root" -d 'null' http://127.0.0.1:8200/v1/transit/keys/kms
+
+      - name: checkout repo
+        uses: actions/checkout@v4
+
+      - name: setup go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.20'
+          cache: false
+
+      - name: setup qemu
+        uses: docker/setup-qemu-action@v3
+
+      - name: setup docker build
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver-opts: network=host
+
+      - name: tag & push docker
+        uses: docker/build-push-action@v5
+        with:
+         context: .
+         push: true
+         tags: localhost:5000/vault-kubernetes-kms
+
+      - name: setup kind
+        uses: helm/kind-action@v1
+        with:
+          node_image: "kindest/node:v1.29.2"
+          config: scripts/kind-config.yaml
diff --git a/Dockerfile b/Dockerfile
@@ -3,7 +3,7 @@ WORKDIR /usr/src/vault-kubernetes-kms
 COPY go.mod go.sum ./
 RUN go mod download && go mod verify
 COPY . . 
-RUN go build -v -o /vault-kubernetes-kms cmd/main.go
+RUN CGO_ENABLED=0 go build -v -o /vault-kubernetes-kms cmd/main.go
 
 # https://github.com/GoogleContainerTools/distroless/issues/1360#issuecomment-1646667145
 FROM gcr.io/distroless/static-debian12@sha256:6dcc833df2a475be1a3d7fc951de90ac91a2cb0be237c7578b88722e48f2e56f

diff --git a/Makefile b/Makefile
@@ -1,5 +1,3 @@
-projectname?=vault-kubernetes-kms
-
 default: help
 
 .PHONY: help
@@ -10,23 +8,27 @@ PHONY: fmt
 fmt: ## format go files
 	gofumpt -w .
 	gci write .
-
-.PHONY: build
-build: ## build plugin and image
-	./scripts/build.sh
 
-.PHONY: run
-run: build ## apply kms k8s manifest
-	./scripts/run.sh
+.PHONY: docs
+docs: ## render docs locally
+	mkdocs serve
+
+PHONY: lint
+lint: ## lint go files
+	golangci-lint run -c .golang-ci.yml
 
-..PHONY: vault
-vault: ## creates vault dev server with transit engine + key
+..PHONY: setup-vault
+setup-vault: ## setup a local vault dev server with transit engine + key
 	./scripts/vault.sh
 
-.PHONY: minikube
-minikube: ## starts minikube
-	./scripts/minikube.sh $(version)
+.PHONY: setup-registry
+setup-registry: ## setup a local docker registry for pulling in kind
+	./scripts/local-registry.sh
 
-PHONY: lint
-lint: ## lint go files
-	golangci-lint run -c .golang-ci.yml
+.PHONY: setup-kind
+setup-kind: ## setup kind cluster with encrpytion provider configured
+	kind delete cluster --name=kms || true 
+	kind create cluster --name=kms --config scripts/kind-config.yaml
+
+.PHONY: setup-local
+setup-local: setup-vault setup-registry setup-kind ## complete local setup
diff --git a/docs/development.md b/docs/development.md
@@ -0,0 +1,228 @@
+# Development 
+
+The following steps describe how to build & run the vault-kubernetes-kms completely locally using `docker`, `vault` & `kind`.
+
+## Requirements
+Obviously you will need all the tools mentioned above installed. Also this setup is only tested on Linux & x86
+
+## Components
+Basically, we will need:
+
+1. A local Vault server initialized & unsealed and with a transit engine enabled aswell as a transit key created.
+2. A local (docker) registry so kind can pull the currently unreleased `vault-kubernetes-kms` image.
+3. A local Kubernetes Cluster (kind) configured to use the local registry aswell as the required settings for the kube-apiservers encryption provider config.
+
+### 1. Local Vault Server
+The following snippets sets up a local vault development server and creates a transit engine as well as a key. This script is located in `scripts/vault.sh` and available via `make setup-vault`
+
+```bash
+#!/usr/bin/bash
+set -x 
+
+# kill any remaining vault instances
+kill $(pgrep -x vault) || true 
+
+# start developemnt vault
+nohup vault server -dev -dev-listen-address=0.0.0.0:8200 -dev-root-token-id=root 2> /dev/null &
+sleep 3
+
+# auth to vault
+export VAULT_ADDR="http://127.0.0.1:8200"
+export VAULT_SKIP_VERIFY="true"
+export VAULT_TOKEN="root"
+
+# enable transit engine
+vault secrets enable transit
+vault write -f transit/keys/kms
+```
+
+### 2. Local Container/Docker Registry
+The following snippets, starts a local container registry, builds the current commits `vault-kubernetes-kms` image and tags & pushes the image to the local registry. This script is localted in `scripts/local-registry.sh` and is available via `make setup-registry`:
+
+```bash
+#!/usr/bin/env bash
+
+set -eu
+
+REGISTRY_NAME=registry
+REGISTRY_PORT=5000
+IMAGE_NAME=vault-kubernetes-kms
+
+echo "====> creating registry container unless it already exists"
+[[ $(docker ps -f "name=${REGISTRY_NAME}" --format '{{.Names}}') == $REGISTRY_NAME ]] || docker run -d --restart=always -p "${REGISTRY_PORT}:5000" --name "${REGISTRY_NAME}" registry:2
+
+echo "====> building container ..."
+docker build --no-cache -t "${IMAGE_NAME}:latest" . 
+
+echo "====> tagging container ..."
+docker tag "${IMAGE_NAME}:latest" "localhost:${REGISTRY_PORT}/${IMAGE_NAME}:latest"
+
+echo "====> pushing container to local registry ...."
+docker push "localhost:${REGISTRY_PORT}/${IMAGE_NAME}:latest"
+
+echo "====> connecting registry to kind ...."
+docker network connect kind "${REGISTRY_NAME}" || true 
+```
+
+### 3. Local Kubernetes Cluster
+Last but not least, we combine the above mentioned tools and consume them with `kind`
+
+The following `kind`-config configured the local running registry, copies the encryption provider config and the `vault-kubernetes-kms` static pod manifest to the Kubernetes host and patches the `kube-apiserver` for using the provided encryption provider config. This can be run via `make setup-kind`, which runs `kind create cluster --name=kms --config scripts/kind-config.yaml` under the hood:
+
+```yaml
+# scripts/kind-config.yaml
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+# add a local docker registry to containerd
+# the registry is run via  a separated docker container
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:5000"]
+    endpoint = ["http://registry:5000"]
+nodes:
+- role: control-plane
+  extraMounts:
+  # mount encryption provider config available on all cp nodes
+  - containerPath: /etc/kubernetes/encryption_provider_config_v2.yaml
+    hostPath: scripts/encryption_provider_config_v2.yml
+    readOnly: true
+    propagation: None
+  # vault-kubernetes-kms as a static Pod
+  - containerPath: /etc/kubernetes/manifests/vault-kubernetes-kms.yaml
+    hostPath: scripts/vault-kubernetes-kms.yml
+    readOnly: true
+    propagation: None
+  # patch kube-apiserver
+  kubeadmConfigPatches:
+    - |
+      kind: ClusterConfiguration
+      apiServer:
+        extraArgs:
+          encryption-provider-config: "/etc/kubernetes/encryption_provider_config_v2.yaml"
+        extraVolumes:
+        - name: encryption-config
+          hostPath: "/etc/kubernetes/encryption_provider_config_v2.yaml"
+          mountPath: "/etc/kubernetes/encryption_provider_config_v2.yaml"
+          readOnly: true
+          pathType: File
+        - name: socket
+          hostPath: "/opt/kms"
+          mountPath: "/opt/kms"
+```
+
+for development purposes, we use the vault dev servers configured root token (`"root"`) as well as the docker ip of the localhost, where the vault server is running (`172.18.0.1`):
+
+```yaml
+# scripts/vault-kubernete-kms.yaml
+apiVersion: v1
+kind: Pod
+metadata:
+  name: vault-kubernetes-kms
+  namespace: kube-system
+spec:
+  priorityClassName: system-node-critical
+  hostNetwork: true
+  containers:
+    - name: vault-kubernetes-kms
+      image: localhost:5000/vault-kubernetes-kms:latest
+      imagePullPolicy: IfNotPresent
+      command:
+        - /vault-kubernetes-kms
+        - --vault-address=http://172.17.0.1:8200    # might needs to be changed according to you docker configuration
+        - --socket=unix:///opt/kms/vaultkms.socket
+        - --vault-token=root                        # pre configured vault root token
+      volumeMounts:
+        # mount /opt/kms host directory
+        - name: kms
+          mountPath: /opt/kms
+  volumes:
+    # mount /opt/kms host directory
+    - name: kms
+      hostPath:
+        path: /opt/kms
+```
+
+## Putting it together
+So if you wanna run all components locally and build the current commits plugin, it would look like this:
+
+```bash
+$> make setup-vault
+$> make setup-registry
+$> make setup-kind
+kind delete cluster --name=kms || true 
+Deleting cluster "kms" ...
+Deleted nodes: ["kms-control-plane"]
+kind create cluster --name=kms --config scripts/kind-config.yaml
+Creating cluster "kms" ...
+ ✓ Ensuring node image (kindest/node:v1.29.2) 🖼
+ ✓ Preparing nodes 📦  
+ ✓ Writing configuration 📜 
+ ✓ Starting control-plane 🕹️ 
+ ✓ Installing CNI 🔌 
+ ✓ Installing StorageClass 💾 
+Set kubectl context to "kind-kms"
+You can now use your cluster with:
+
+# testing kubectl
+$> kubectl get pod -A                                                             
+NAMESPACE            NAME                                        READY   STATUS    RESTARTS   AGE
+kube-system          coredns-76f75df574-7pzq4                    1/1     Running   0          17m
+kube-system          coredns-76f75df574-pkqrj                    1/1     Running   0          17m
+kube-system          etcd-kms-control-plane                      1/1     Running   0          17m
+kube-system          kindnet-w2hgj                               1/1     Running   0          17m
+kube-system          kube-apiserver-kms-control-plane            1/1     Running   0          17m
+kube-system          kube-controller-manager-kms-control-plane   1/1     Running   0          17m
+kube-system          kube-proxy-w66mx                            1/1     Running   0          17m
+kube-system          kube-scheduler-kms-control-plane            1/1     Running   0          17m
+kube-system          vault-kubernetes-kms-kms-control-plane      1/1     Running   0          17m
+local-path-storage   local-path-provisioner-7577fdbbfb-rmqq8     1/1     Running   0          17m
+
+# creating a kubernetes secret
+$> kubectl create secret generic secret -n default --from-literal=key=value
+secret/secret created
+
+# checking encryption value in etcd
+$> kubectl -n kube-system exec etcd-kms-control-plane -- sh -c "ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 \
+    --cert /etc/kubernetes/pki/etcd/server.crt \
+    --key /etc/kubernetes/pki/etcd/server.key \
+    --cacert /etc/kubernetes/pki/etcd/ca.crt \
+    get /registry/secrets/default/secret" | hexdump -C
+00000000  2f 72 65 67 69 73 74 72  79 2f 73 65 63 72 65 74  |/registry/secret|
+00000010  73 2f 64 65 66 61 75 6c  74 2f 73 65 63 72 65 74  |s/default/secret|
+00000020  2d 65 6e 63 72 79 70 74  65 64 0a 6b 38 73 3a 65  |-encrypted.k8s:e|
+00000030  6e 63 3a 6b 6d 73 3a 76  32 3a 76 61 75 6c 74 2d  |nc:kms:v2:vault-|
+00000040  6b 75 62 65 72 6e 65 74  65 73 2d 6b 6d 73 3a 0a  |kubernetes-kms:.|
+00000050  a4 02 7f fe e1 bb 63 29  71 62 b6 1f c0 be d5 a0  |......c)qb......|
+00000060  a8 38 0b e6 a1 bc 4b bb  16 ff 3f d3 3f 14 e4 be  |.8....K...?.?...|
+00000070  7e fa 53 de d5 06 75 08  3a fd 5f fb e9 a3 b1 29  |~.S...u.:._....)|
+00000080  e2 9f 26 1c ef bb 1b 24  37 bc f3 ab 9c df 46 c4  |..&....$7.....F.|
+00000090  8f 47 33 e5 c0 76 54 3b  e7 f4 3b da 0d bf 80 e0  |.G3..vT;..;.....|
+000000a0  52 88 cd 1a 6f c6 ec 7f  bb 51 4b ef 0c c7 b6 8f  |R...o....QK.....|
+000000b0  31 2d 6b 96 3d 37 ee cb  f0 56 83 40 d8 b4 21 75  |1-k.=7...V.@..!u|
+000000c0  31 78 e7 ab ec 5f 6e f7  bf 84 86 34 2a aa 65 1b  |1x..._n....4*.e.|
+000000d0  8a 2b ce 6c ae 6f b6 df  11 5b ec 14 9d b9 00 74  |.+.l.o...[.....t|
+000000e0  9d 0c 01 11 c4 67 48 67  3d d3 8f 58 1a 0d da 34  |.....gHg=..X...4|
+000000f0  0d 55 19 91 cc 7e db c3  36 a2 6d 2f ea 28 10 ab  |.U...~..6.m/.(..|
+00000100  9b 1e 71 a9 d4 b1 74 6b  2f cc ef aa 30 d9 1a b8  |..q...tk/...0...|
+00000110  68 30 3b 5b c5 3a 32 69  6a 75 4d 43 68 1f 33 23  |h0;[.:2ijuMCh.3#|
+00000120  af 56 8c 15 c9 17 cb 8a  46 fc 9f 5a 24 da 25 16  |.V......F..Z$.%.|
+00000130  15 31 ce 41 59 6b b8 c6  7d 5e b3 ee 07 a7 65 3b  |.1.AYk..}^....e;|
+00000140  a8 f2 8a ab e7 d0 37 bc  9c e6 e6 33 71 57 c5 6c  |......7....3qW.l|
+00000150  09 ff e9 65 c9 8c 9f aa  1c e2 df a4 ad fc a0 02  |...e............|
+00000160  2b 6d 93 5e 44 20 64 28  d7 3f e1 98 eb 84 ab 22  |+m.^D d(.?....."|
+00000170  82 92 7a b6 b2 b8 12 0a  31 37 31 31 32 34 32 33  |..z.....17112423|
+00000180  36 34 1a 59 76 61 75 6c  74 3a 76 31 3a 6d 42 41  |64.Yvault:v1:mBA|
+00000190  4a 47 56 56 35 72 46 78  36 47 4c 4f 62 33 46 50  |JGVV5rFx6GLOb3FP|
+000001a0  37 4a 38 73 5a 79 4a 38  2f 68 36 61 48 2b 46 57  |7J8sZyJ8/h6aH+FW|
+000001b0  55 46 2f 67 53 68 30 65  41 31 4e 51 45 47 6e 30  |UF/gSh0eA1NQEGn0|
+000001c0  5a 30 38 66 6a 59 45 53  30 4c 31 79 35 45 49 50  |Z08fjYES0L1y5EIP|
+000001d0  33 67 4c 72 77 61 35 4b  61 44 35 43 63 28 01 0a  |3gLrwa5KaD5Cc(..|
+000001e0
+
+# receiving the secret
+$>  kubectl get secret secret -o json | jq '.data | map_values(@base64d)'
+{
+  "key": "value"
+}
+```
+