Boost pods before they are submitted by using a mutating webhook

.github/workflows/kind-e2e.yaml

@@ -22,7 +22,7 @@ jobs:
         - v1.27.x
         - v1.28.x
         os:
-        - ubuntu-20.04 # Ubuntu 20.04 uses cgroup v1
+        # - ubuntu-20.04 # Ubuntu 20.04 uses cgroup v1 # TODO: temporarily disabled because of openssl commands incompatibility (1.1.1 version vs 3.x)
         - ubuntu-22.04 # Ubuntu 22.04 uses cgroup v2
     runs-on: ${{ matrix.os }}
 
@@ -51,12 +51,14 @@ jobs:
 
     - name: Install k8s-pod-cpu-booster
       run: |
-        ko apply -f config/
+        make --directory config/ mutating-webhook-certs
+        kustomize build config/ | ko apply -f -
 
     - name: Wait for Ready
       run: |
-        echo "Waiting for Pods to become ready"
-        kubectl wait pod --for=condition=Ready -n pod-cpu-booster -l name=pod-cpu-booster
+        echo "Waiting for k8s-pod-cpu-booster items to become ready"
+        kubectl wait pod --for=condition=Ready -n pod-cpu-booster -l app=pod-cpu-boost-reset
+        kubectl wait pod --for=condition=Ready -n pod-cpu-booster -l app=mutating-webhook
         sleep 5 # because readiness probe is not accurate (Ready != informer is started), but sleeping is enough for now
 
     - name: Run e2e Tests

README.md

@@ -15,19 +15,23 @@ Between startup and `Ready` status, the container benefits from a CPU boost (x10
 
 ## How does it work?
 
-It is deployed as a controller on every node (with a `DaemonSet`). It listens for every pod update; if a pod has `norbjd.github.io/k8s-pod-cpu-booster-enabled: "true"` label: it boosts the CPU at pod startup, and reset the CPU limit when the pod is ready.
+It is deployed in two parts:
 
-The CPU boost can be configured with `norbjd.github.io/k8s-pod-cpu-booster-multiplier` annotation:
+- a mutating webhook boosting the CPU of pods with `norbjd.github.io/k8s-pod-cpu-booster-enabled: "true"` label, before they are submitted to k8s API
+- a controller listening for every update of pods with `norbjd.github.io/k8s-pod-cpu-booster-enabled: "true"` label; when a pod is ready, it will reset its CPU limit
 
-- if specified, it will increase the CPU limit by `x`, where `x` is the value of the annotation (must be an unsigned integer)
+The CPU boost can be configured with `norbjd.github.io/k8s-pod-cpu-booster-multiplier` label:
+
+- if specified, it will increase the CPU limit by `x`, where `x` is the value of the label (must be an unsigned integer)
 - if unspecified or invalid, it will increase the CPU limit by the default value (`10`)
 
 ## Install
 
 Use `ko`. Example on a `kind` cluster:
 
 ```sh
-KO_DOCKER_REPO=kind.local ko apply -f config/
+make --directory config/ mutating-webhook-certs # generates self-signed certificates for the webhook
+kustomize build config/ | KO_DOCKER_REPO=kind.local ko apply -f -
 ```
 
 ## Test/Demo
@@ -48,25 +52,26 @@ kind load docker-image python:3.11-alpine
 Install `k8s-pod-cpu-booster`:
 
 ```sh
-KO_DOCKER_REPO=kind.local ko apply -f config/
+make --directory config/ mutating-webhook-certs # generates self-signed certificates for the webhook
+kustomize build config/ | KO_DOCKER_REPO=kind.local ko apply -f -
 ```
 
-Start two similar pods with low CPU limits and running `python -m http.server`, with a readiness probe configured to check when the http server is started. The only differences are the name (obviously), and the annotation `norbjd.github.io/k8s-pod-cpu-booster-enabled`:
+Start two similar pods with low CPU limits and running `python -m http.server`, with a readiness probe configured to check when the http server is started. The only differences are the name (obviously), and the label `norbjd.github.io/k8s-pod-cpu-booster-enabled`:
 
 ```diff
 --- examples/pod-no-boost.yaml
 +++ examples/pod-with-default-boost.yaml
 @@ -4 +4,3 @@
 -  name: pod-no-boost
-+  annotations:
++  labels:
 +    norbjd.github.io/k8s-pod-cpu-booster-enabled: "true"
 +  name: pod-with-default-boost
 ```
 
 > [!NOTE]
-> The CPU boost multiplier can also be configured (see [`pod-with-small-boost.yaml`](https://github.com/norbjd/k8s-pod-cpu-booster/blob/main/examples/pod-with-small-boost.yaml)) by using the `norbjd.github.io/k8s-pod-cpu-booster-multiplier` annotation.
+> The CPU boost multiplier can also be configured (see [`pod-with-small-boost.yaml`](https://github.com/norbjd/k8s-pod-cpu-booster/blob/main/examples/pod-with-small-boost.yaml)) by using the `norbjd.github.io/k8s-pod-cpu-booster-multiplier` label.
 
-As a result, the pod `pod-with-default-boost` (with the annotation) will benefit from a CPU boost, but `pod-no-boost` won't:
+As a result, the pod `pod-with-default-boost` (with the label) will benefit from a CPU boost, but `pod-no-boost` won't:
 
 ```sh
 kubectl apply -f examples/pod-no-boost.yaml -f examples/pod-with-default-boost.yaml
@@ -101,7 +106,8 @@ Cleanup:
 ```sh
 kubectl delete -f examples/pod-no-boost.yaml -f examples/pod-with-default-boost.yaml
 
-KO_DOCKER_REPO=kind.local ko delete -f config/
+kustomize build config/ | KO_DOCKER_REPO=kind.local ko delete -f -
+make --directory config/ remove-certs
 
 kind delete cluster
 ```

cmd/webhook/main.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"flag"
+
+	"github.com/norbjd/k8s-pod-cpu-booster/pkg/webhook"
+	"k8s.io/klog/v2"
+)
+
+func main() {
+	klog.InitFlags(nil)
+
+	var port uint
+	var pathToCertFile string
+	var pathToKeyFile string
+
+	flag.UintVar(&port, "port", 8443, "listening port")
+	flag.StringVar(&pathToCertFile, "cert", "", "path to cert file")
+	flag.StringVar(&pathToKeyFile, "key", "", "path to key file")
+
+	flag.Parse()
+
+	err := webhook.Run(port, pathToCertFile, pathToKeyFile)
+	if err != nil {
+		klog.Fatal(err)
+	}
+}

config/.gitignore

@@ -0,0 +1,2 @@
+*.crt
+*.key

config/GNUmakefile

@@ -0,0 +1,13 @@
+# namespace is used in the certificate's CN and SAN
+NAMESPACE := $(shell grep 'namespace: ' kustomization.yaml | cut -d ' ' -f 2)
+
+mutating-webhook-certs:
+	openssl genrsa -out ca.key 4096
+	openssl req -new -x509 -key ca.key -out ca.crt -days 3650 -nodes -subj "/CN=my-self-signed-ca"
+	openssl req -x509 -CA ca.crt -CAkey ca.key -keyout tls.key -out tls.crt -sha256 -days 3650 -nodes -subj "/CN=mutating-webhook.${NAMESPACE}.svc" -addext "subjectAltName = DNS:mutating-webhook.${NAMESPACE}.svc"
+
+remove-certs:
+	rm -f *.key *.crt
+
+build:
+	kustomize build .

config/kustomization.yaml

@@ -0,0 +1,26 @@
+---
+namespace: pod-cpu-booster
+resources:
+  - mutating-webhook.yaml
+  - namespace.yaml
+  - pod-cpu-boost-reset.yaml
+secretGenerator:
+  - name: mutating-webhook-certs
+    options:
+      disableNameSuffixHash: true
+    type: kubernetes.io/tls
+    files:
+      - ca.crt
+      - tls.crt
+      - tls.key
+replacements:
+  - source:
+      kind: Secret
+      name: mutating-webhook-certs
+      fieldPath: "data.[ca.crt]"
+    targets:
+      - select:
+          kind: MutatingWebhookConfiguration
+          name: k8s-pod-cpu-booster
+        fieldPaths:
+          - webhooks.[name=k8s-pod-cpu-booster.norbjd.github.io].clientConfig.caBundle

config/mutating-webhook.yaml

@@ -0,0 +1,75 @@
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: k8s-pod-cpu-booster
+webhooks:
+  - name: k8s-pod-cpu-booster.norbjd.github.io
+    clientConfig:
+      caBundle: "" # will be overridden by kustomization replacement
+      service: # namespace field is overridden by the namespace defined in kustomization.yaml
+        name: mutating-webhook
+        path: /mutate
+    objectSelector:
+      matchExpressions:
+        # we don't want that creation of mutating-webhook pods triggers the webhook (otherwise pods won't start)
+        - key: app
+          operator: NotIn
+          values:
+            - mutating-webhook
+        - key: norbjd.github.io/k8s-pod-cpu-booster-enabled
+          operator: In
+          values:
+            - "true"
+    rules:
+      - apiGroups: [""]
+        apiVersions: ["v1"]
+        resources: ["pods"]
+        operations: ["CREATE"]
+        scope: Namespaced
+    sideEffects: None
+    admissionReviewVersions: ["v1"]
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mutating-webhook
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: mutating-webhook
+  template:
+    metadata:
+      labels:
+        app: mutating-webhook
+    spec:
+      containers:
+        - name: mutating-webhook
+          image: ko://github.com/norbjd/k8s-pod-cpu-booster/cmd/webhook
+          args:
+            - -v=9
+            - -port=8443
+            - -cert=/etc/certs/tls.crt
+            - -key=/etc/certs/tls.key
+          ports:
+            - containerPort: 8443
+          volumeMounts:
+            - name: certs
+              mountPath: /etc/certs
+              readOnly: true
+      volumes:
+        - name: certs
+          secret:
+            secretName: mutating-webhook-certs
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mutating-webhook
+spec:
+  selector:
+    app: mutating-webhook
+  ports:
+    - port: 443
+      targetPort: 8443

config/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: doesnt-matter # will be overridden by kustomize

config/pod-cpu-boost-reset.yaml

@@ -0,0 +1,59 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: pod-cpu-boost-reset
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: pod-cpu-boost-reset
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - list
+      - watch
+      - get
+      - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: pod-cpu-boost-reset
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: pod-cpu-boost-reset
+subjects:
+  - kind: ServiceAccount
+    name: pod-cpu-boost-reset
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: pod-cpu-boost-reset
+spec:
+  replicas: 1 # for now, we don't support multiple replicas
+  selector:
+    matchLabels:
+      app: pod-cpu-boost-reset
+  template:
+    metadata:
+      labels:
+        app: pod-cpu-boost-reset
+    spec:
+      containers:
+      - name: pod-cpu-boost-reset
+        image: ko://github.com/norbjd/k8s-pod-cpu-booster/cmd/informer
+        resources:
+          limits:
+            cpu: 100m
+            memory: 100Mi
+          requests:
+            cpu: 100m
+            memory: 100Mi
+      serviceAccountName: pod-cpu-boost-reset
+      terminationGracePeriodSeconds: 0 # TODO: change for production environments

examples/deployment-with-default-boost.yaml

@@ -9,9 +9,8 @@ spec:
       app: deployment-with-default-boost
   template:
     metadata:
-      annotations:
-        norbjd.github.io/k8s-pod-cpu-booster-enabled: "true"
       labels:
+        norbjd.github.io/k8s-pod-cpu-booster-enabled: "true"
         app: deployment-with-default-boost
     spec:
       containers: