Merge pull request #10427 from medyagh/auto-pause

auto-pause addon: automatically pause Kubernetes when not in use

.gitignore

@@ -26,6 +26,7 @@ _testmain.go
 *.test
 *.prof
 
+/deploy/kicbase/auto-pause
 /out
 /_gopath
 

Makefile

@@ -648,7 +648,7 @@ KICBASE_ARCH = linux/arm64,linux/amd64
 KICBASE_IMAGE_REGISTRIES ?= $(REGISTRY)/kicbase:$(KIC_VERSION) kicbase/stable:$(KIC_VERSION)
 
 .PHONY: push-kic-base-image 
-push-kic-base-image: docker-multi-arch-builder ## Push multi-arch local/kicbase:latest to all remote registries
+push-kic-base-image: deploy/kicbase/auto-pause docker-multi-arch-builder ## Push multi-arch local/kicbase:latest to all remote registries
 ifdef AUTOPUSH
 	docker login gcr.io/k8s-minikube
 	docker login docker.pkg.github.com
@@ -812,6 +812,10 @@ site: site/themes/docsy/assets/vendor/bootstrap/package.js out/hugo/hugo ## Serv
 out/mkcmp:
 	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/mkcmp/main.go
 
+.PHONY: deploy/kicbase/auto-pause # auto pause binary to be used for kic image work arround for not passing the whole repo as docker context
+deploy/kicbase/auto-pause: $(SOURCE_GENERATED) $(SOURCE_FILES)
+	GOOS=linux GOARCH=$(GOARCH) go build -o $@ cmd/auto-pause/auto-pause.go
+
 .PHONY: out/performance-bot
 out/performance-bot:
 	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/pr-bot/bot.go

cmd/auto-pause/auto-pause.go

@@ -0,0 +1,123 @@
+/*
+Copyright 2021 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"sync"
+	"time"
+
+	"k8s.io/minikube/pkg/minikube/cluster"
+	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/exit"
+	"k8s.io/minikube/pkg/minikube/out"
+	"k8s.io/minikube/pkg/minikube/reason"
+	"k8s.io/minikube/pkg/minikube/style"
+)
+
+var unpauseRequests = make(chan struct{})
+var done = make(chan struct{})
+var mu sync.Mutex
+
+// TODO: initialize with current state (handle the case that user enables auto-pause after it is already paused)
+var runtimePaused = false
+var version = "0.0.1"
+
+// TODO: #10597 make this configurable to support containerd/cri-o
+var runtime = "docker"
+
+func main() {
+	// TODO: #10595 make this configurable
+	const interval = time.Minute * 1
+	// channel for incoming messages
+	go func() {
+		for {
+			// On each iteration new timer is created
+			select {
+			// TODO: #10596 make it memory-leak proof
+			case <-time.After(interval):
+				runPause()
+			case <-unpauseRequests:
+				fmt.Printf("Got request\n")
+				if runtimePaused {
+					runUnpause()
+				}
+
+				done <- struct{}{}
+			}
+		}
+	}()
+
+	http.HandleFunc("/", handler) // each request calls handler
+	fmt.Printf("Starting auto-pause server %s at port 8080 \n", version)
+	log.Fatal(http.ListenAndServe("0.0.0.0:8080", nil))
+}
+
+// handler echoes the Path component of the requested URL.
+func handler(w http.ResponseWriter, r *http.Request) {
+	unpauseRequests <- struct{}{}
+	<-done
+	fmt.Fprintf(w, "allow")
+}
+
+func runPause() {
+	mu.Lock()
+	defer mu.Unlock()
+	if runtimePaused {
+		return
+	}
+
+	r := command.NewExecRunner(true)
+
+	cr, err := cruntime.New(cruntime.Config{Type: runtime, Runner: r})
+	if err != nil {
+		exit.Error(reason.InternalNewRuntime, "Failed runtime", err)
+	}
+
+	uids, err := cluster.Pause(cr, r, []string{"kube-system"})
+	if err != nil {
+		exit.Error(reason.GuestPause, "Pause", err)
+	}
+
+	runtimePaused = true
+
+	out.Step(style.Unpause, "Paused {{.count}} containers", out.V{"count": len(uids)})
+}
+
+func runUnpause() {
+	fmt.Println("unpausing...")
+	mu.Lock()
+	defer mu.Unlock()
+
+	r := command.NewExecRunner(true)
+
+	cr, err := cruntime.New(cruntime.Config{Type: runtime, Runner: r})
+	if err != nil {
+		exit.Error(reason.InternalNewRuntime, "Failed runtime", err)
+	}
+
+	uids, err := cluster.Unpause(cr, r, nil)
+	if err != nil {
+		exit.Error(reason.GuestUnpause, "Unpause", err)
+	}
+	runtimePaused = false
+
+	out.Step(style.Unpause, "Unpaused {{.count}} containers", out.V{"count": len(uids)})
+}

cmd/minikube/cmd/status.go

@@ -379,7 +379,14 @@ func nodeStatus(api libmachine.API, cc config.ClusterConfig, n config.Node) (*St
 		return st, nil
 	}
 
-	hostname, _, port, err := driver.ControlPlaneEndpoint(&cc, &n, host.DriverName)
+	var hostname string
+	var port int
+	if cc.Addons["auto-pause"] {
+		hostname, _, port, err = driver.AutoPauseProxyEndpoint(&cc, &n, host.DriverName)
+	} else {
+		hostname, _, port, err = driver.ControlPlaneEndpoint(&cc, &n, host.DriverName)
+	}
+
 	if err != nil {
 		klog.Errorf("forwarded endpoint: %v", err)
 		st.Kubeconfig = Misconfigured

deploy/addons/auto-pause/auto-pause.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Auto Pause Service
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/auto-pause
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

deploy/addons/auto-pause/auto-pause.yaml.tmpl

@@ -0,0 +1,47 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: auto-pause
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: auto-pause-proxy
+  namespace: auto-pause
+  labels:
+    app: auto-pause-proxy
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: auto-pause-proxy
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app: auto-pause-proxy
+    spec:
+      volumes:
+      - name: ha-cfg
+        hostPath:
+          path: /var/lib/minikube/haproxy.cfg
+          type: File
+      - name: lua-script
+        hostPath:
+          path: /var/lib/minikube/unpause.lua
+          type: File
+      containers:
+        - name: auto-pause
+          image: "haproxy:2.3.5-alpine"
+          ports:
+            - name: https
+              containerPort: 6443
+              hostPort: 32443
+              protocol: TCP
+          volumeMounts:
+          - name: ha-cfg
+            mountPath: /usr/local/etc/haproxy/haproxy.cfg
+            readOnly: true
+          - name: lua-script
+            mountPath: /etc/haproxy/unpause.lua

deploy/addons/auto-pause/haproxy.cfg

@@ -0,0 +1,37 @@
+#---------------------------------------------------------------------
+# Configure HAProxy for Kubernetes API Server
+#---------------------------------------------------------------------
+listen stats
+  bind    *:9000
+  mode    http
+  stats   enable
+  stats   hide-version
+  stats   uri       /
+  stats   refresh   30s
+  option  httplog
+
+# change haproxy.cfg file with the following
+global
+   lua-load /etc/haproxy/unpause.lua
+
+############## Configure HAProxy Secure Frontend #############
+frontend k8s-api-https-proxy
+    bind *:6443
+    mode tcp
+    tcp-request inspect-delay 5s
+    tcp-request content accept if { req.ssl_hello_type 1 }
+    default_backend k8s-api-https
+############## Configure HAProxy SecureBackend #############
+backend k8s-api-https
+    balance roundrobin
+    mode tcp
+    #tcp-request inspect-delay 10s
+    #tcp-request content lua.foo_action
+    tcp-request inspect-delay 10s
+    tcp-request content lua.unpause 192.168.49.2 8080
+    tcp-request content reject if { var(req.blocked) -m bool }
+    option tcplog
+    option tcp-check
+    default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
+    server k8s-api-1 192.168.49.2:8443 check
+

deploy/addons/auto-pause/unpause.lua

@@ -0,0 +1,57 @@
+local function unpause(txn, addr, port)
+    if not addr then addr = '127.0.0.1' end
+    if not port then port = 5000 end
+
+    -- Set up a request to the service
+    local hdrs = {
+        [1] = string.format('host: %s:%s', addr, port),
+        [2] = 'accept: */*',
+        [3] = 'connection: close'
+    }
+
+    local req = {
+        [1] = string.format('GET /%s HTTP/1.1', tostring(txn.f:src())),
+        [2] = table.concat(hdrs, '\r\n'),
+        [3] = '\r\n'
+    }
+
+    req = table.concat(req,  '\r\n')
+
+    -- Use core.tcp to get an instance of the Socket class
+    local socket = core.tcp()
+    socket:settimeout(5)
+
+    -- Connect to the service and send the request
+    if socket:connect(addr, port) then
+        if socket:send(req) then
+            -- Skip response headers
+            while true do
+                local line, _ = socket:receive('*l')
+
+                if not line then break end
+                if line == '' then break end
+            end
+
+            -- Get response body, if any
+            local content = socket:receive('*a')
+
+            -- Check if this request should be allowed
+            if content and content == 'allow' then
+                txn:set_var('req.blocked', false)
+                return
+            end
+        else
+            core.Alert('Could not connect to IP Checker server (send)')
+        end
+
+        socket:close()
+    else
+        core.Alert('Could not connect to IP Checker server (connect)')
+    end
+
+    -- The request should be blocked
+    txn:set_var('req.blocked', true)
+end
+
+core.register_action('unpause', {'tcp-req'}, unpause, 2)
+

deploy/kicbase/Dockerfile

@@ -28,6 +28,8 @@ COPY 10-network-security.conf /etc/sysctl.d/10-network-security.conf
 COPY 11-tcp-mtu-probing.conf /etc/sysctl.d/11-tcp-mtu-probing.conf
 COPY clean-install /usr/local/bin/clean-install
 COPY entrypoint /usr/local/bin/entrypoint
+# must first run make out/auto-pause
+COPY auto-pause /usr/local/bin/auto-pause
 
 # Install dependencies, first from apt, then from release tarballs.
 # NOTE: we use one RUN to minimize layers.