Add new driver "SSH" to bootstrap generic minkube clusters over ssh

cmd/minikube/cmd/delete.go

@@ -309,7 +309,7 @@ func deleteProfile(profile *config.Profile) error {
 		return DeletionError{Err: delErr, Errtype: MissingProfile}
 	}
 
-	if err == nil && driver.BareMetal(cc.Driver) {
+	if err == nil && (driver.BareMetal(cc.Driver) || driver.IsSSH(cc.Driver)) {
 		if err := uninstallKubernetes(api, *cc, cc.Nodes[0], viper.GetString(cmdcfg.Bootstrapper)); err != nil {
 			deletionError, ok := err.(DeletionError)
 			if ok {

cmd/minikube/cmd/start.go

@@ -306,7 +306,7 @@ func provisionWithDriver(cmd *cobra.Command, ds registry.DriverState, existing *
 		os.Exit(0)
 	}
 
-	if driver.IsVM(driverName) {
+	if driver.IsVM(driverName) && !driver.IsSSH(driverName) {
 		url, err := download.ISO(viper.GetStringSlice(isoURL), cmd.Flags().Changed(isoURL))
 		if err != nil {
 			return node.Starter{}, errors.Wrap(err, "Failed to cache ISO")
@@ -851,7 +851,7 @@ func validateUser(drvName string) {
 
 // memoryLimits returns the amount of memory allocated to the system and hypervisor, the return value is in MiB
 func memoryLimits(drvName string) (int, int, error) {
-	info, cpuErr, memErr, diskErr := machine.CachedHostInfo()
+	info, cpuErr, memErr, diskErr := machine.LocalHostInfo()
 	if cpuErr != nil {
 		klog.Warningf("could not get system cpu info while verifying memory limits, which might be okay: %v", cpuErr)
 	}

cmd/minikube/cmd/start_flags.go

@@ -110,6 +110,12 @@ const (
 	network                 = "network"
 	startNamespace          = "namespace"
 	trace                   = "trace"
+	sshIPAddress            = "ip-address"
+	sshSSHUser              = "ssh-user"
+	sshSSHKey               = "ssh-key"
+	sshSSHPort              = "ssh-port"
+	defaultSSHUser          = "root"
+	defaultSSHPort          = 22
 )
 
 var (
@@ -221,6 +227,12 @@ func initNetworkingFlags() {
 	startCmd.Flags().String(serviceCIDR, constants.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.")
 	startCmd.Flags().StringArrayVar(&config.DockerEnv, "docker-env", nil, "Environment variables to pass to the Docker daemon. (format: key=value)")
 	startCmd.Flags().StringArrayVar(&config.DockerOpt, "docker-opt", nil, "Specify arbitrary flags to pass to the Docker daemon. (format: key=value)")
+
+	// ssh
+	startCmd.Flags().String(sshIPAddress, "", "IP address (ssh driver only)")
+	startCmd.Flags().String(sshSSHUser, defaultSSHUser, "SSH user (ssh driver only)")
+	startCmd.Flags().String(sshSSHKey, "", "SSH key (ssh driver only)")
+	startCmd.Flags().Int(sshSSHPort, defaultSSHPort, "SSH port (ssh driver only)")
 }
 
 // ClusterFlagValue returns the current cluster name based on flags
@@ -335,6 +347,10 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k
 			NatNicType:              viper.GetString(natNicType),
 			StartHostTimeout:        viper.GetDuration(waitTimeout),
 			ExposedPorts:            viper.GetStringSlice(ports),
+			IPAddress:               viper.GetString(sshIPAddress),
+			SSHUser:                 viper.GetString(sshSSHUser),
+			SSHKey:                  viper.GetString(sshSSHKey),
+			SSHPort:                 viper.GetInt(sshSSHPort),
 			KubernetesConfig: config.KubernetesConfig{
 				KubernetesVersion:      k8sVersion,
 				ClusterName:            ClusterFlagValue(),

pkg/drivers/ssh/ssh.go

@@ -0,0 +1,243 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ssh
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path"
+	"strconv"
+	"time"
+
+	"github.com/docker/machine/libmachine/drivers"
+	"github.com/docker/machine/libmachine/engine"
+	"github.com/docker/machine/libmachine/log"
+	"github.com/docker/machine/libmachine/mcnutils"
+	"github.com/docker/machine/libmachine/state"
+	"github.com/pkg/errors"
+	"k8s.io/klog/v2"
+	pkgdrivers "k8s.io/minikube/pkg/drivers"
+	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/sysinit"
+)
+
+// Driver is a driver designed to run kubeadm w/o VM management.
+// https://minikube.sigs.k8s.io/docs/reference/drivers/ssh/
+type Driver struct {
+	*drivers.BaseDriver
+	*pkgdrivers.CommonDriver
+	EnginePort int
+	SSHKey     string
+	runtime    cruntime.Manager
+	exec       command.Runner
+}
+
+// Config is configuration for the SSH driver
+type Config struct {
+	MachineName      string
+	StorePath        string
+	ContainerRuntime string
+}
+
+const (
+	defaultTimeout = 15 * time.Second
+)
+
+// NewDriver creates and returns a new instance of the driver
+func NewDriver(c Config) *Driver {
+	d := &Driver{
+		EnginePort: engine.DefaultPort,
+		BaseDriver: &drivers.BaseDriver{
+			MachineName: c.MachineName,
+			StorePath:   c.StorePath,
+		},
+	}
+	runner := command.NewSSHRunner(d)
+	runtime, err := cruntime.New(cruntime.Config{Type: c.ContainerRuntime, Runner: runner})
+	// Libraries shouldn't panic, but there is no way for drivers to return error :(
+	if err != nil {
+		klog.Fatalf("unable to create container runtime: %v", err)
+	}
+	d.runtime = runtime
+	d.exec = runner
+	return d
+}
+
+// DriverName returns the name of the driver
+func (d *Driver) DriverName() string {
+	return "ssh"
+}
+
+func (d *Driver) GetSSHHostname() (string, error) {
+	return d.GetIP()
+}
+
+func (d *Driver) GetSSHUsername() string {
+	return d.SSHUser
+}
+
+func (d *Driver) GetSSHKeyPath() string {
+	return d.SSHKeyPath
+}
+
+func (d *Driver) PreCreateCheck() error {
+	if d.SSHKey != "" {
+		if _, err := os.Stat(d.SSHKey); os.IsNotExist(err) {
+			return fmt.Errorf("SSH key does not exist: %q", d.SSHKey)
+		}
+
+		// TODO: validate the key is a valid key
+	}
+
+	return nil
+}
+
+func (d *Driver) Create() error {
+	if d.SSHKey == "" {
+		log.Info("No SSH key specified. Assuming an existing key at the default location.")
+	} else {
+		log.Info("Importing SSH key...")
+
+		d.SSHKeyPath = d.ResolveStorePath(path.Base(d.SSHKey))
+		if err := copySSHKey(d.SSHKey, d.SSHKeyPath); err != nil {
+			return err
+		}
+
+		if err := copySSHKey(d.SSHKey+".pub", d.SSHKeyPath+".pub"); err != nil {
+			log.Infof("Couldn't copy SSH public key : %s", err)
+		}
+	}
+
+	if d.runtime.Name() == "Docker" {
+		if _, err := d.exec.RunCmd(exec.Command("sudo", "usermod", "-aG", "docker", d.GetSSHUsername())); err != nil {
+			return errors.Wrap(err, "usermod")
+		}
+	}
+
+	log.Debugf("IP: %s", d.IPAddress)
+
+	return nil
+}
+
+func (d *Driver) GetURL() (string, error) {
+	if err := drivers.MustBeRunning(d); err != nil {
+		return "", err
+	}
+
+	ip, err := d.GetIP()
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("tcp://%s", net.JoinHostPort(ip, strconv.Itoa(d.EnginePort))), nil
+}
+
+func (d *Driver) GetState() (state.State, error) {
+	address := net.JoinHostPort(d.IPAddress, strconv.Itoa(d.SSHPort))
+
+	_, err := net.DialTimeout("tcp", address, defaultTimeout)
+	if err != nil {
+		return state.Stopped, nil
+	}
+
+	return state.Running, nil
+}
+
+// Start a host
+func (d *Driver) Start() error {
+	return nil
+}
+
+// Stop a host gracefully, including any containers that we are managing.
+func (d *Driver) Stop() error {
+	if err := sysinit.New(d.exec).Stop("kubelet"); err != nil {
+		klog.Warningf("couldn't stop kubelet. will continue with stop anyways: %v", err)
+		if err := sysinit.New(d.exec).ForceStop("kubelet"); err != nil {
+			klog.Warningf("couldn't force stop kubelet. will continue with stop anyways: %v", err)
+		}
+	}
+	containers, err := d.runtime.ListContainers(cruntime.ListOptions{})
+	if err != nil {
+		return errors.Wrap(err, "containers")
+	}
+	if len(containers) > 0 {
+		if err := d.runtime.StopContainers(containers); err != nil {
+			return errors.Wrap(err, "stop containers")
+		}
+	}
+	klog.Infof("ssh driver is stopped!")
+	return nil
+}
+
+// Restart a host
+func (d *Driver) Restart() error {
+	if err := sysinit.New(d.exec).Restart("kubelet"); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Kill stops a host forcefully, including any containers that we are managing.
+func (d *Driver) Kill() error {
+	if err := sysinit.New(d.exec).ForceStop("kubelet"); err != nil {
+		klog.Warningf("couldn't force stop kubelet. will continue with kill anyways: %v", err)
+	}
+
+	// First try to gracefully stop containers
+	containers, err := d.runtime.ListContainers(cruntime.ListOptions{})
+	if err != nil {
+		return errors.Wrap(err, "containers")
+	}
+	if len(containers) == 0 {
+		return nil
+	}
+	// Try to be graceful before sending SIGKILL everywhere.
+	if err := d.runtime.StopContainers(containers); err != nil {
+		return errors.Wrap(err, "stop")
+	}
+
+	containers, err = d.runtime.ListContainers(cruntime.ListOptions{})
+	if err != nil {
+		return errors.Wrap(err, "containers")
+	}
+	if len(containers) == 0 {
+		return nil
+	}
+	if err := d.runtime.KillContainers(containers); err != nil {
+		return errors.Wrap(err, "kill")
+	}
+	return nil
+}
+
+func (d *Driver) Remove() error {
+	return nil
+}
+
+func copySSHKey(src, dst string) error {
+	if err := mcnutils.CopyFile(src, dst); err != nil {
+		return fmt.Errorf("unable to copy ssh key: %s", err)
+	}
+
+	if err := os.Chmod(dst, 0600); err != nil {
+		return fmt.Errorf("unable to set permissions on the ssh key: %s", err)
+	}
+
+	return nil
+}

pkg/minikube/cluster/ip.go

@@ -40,6 +40,12 @@ func HostIP(host *host.Host, clusterName string) (net.IP, error) {
 		return oci.RoutableHostIPFromInside(oci.Docker, clusterName, host.Name)
 	case driver.Podman:
 		return oci.RoutableHostIPFromInside(oci.Podman, clusterName, host.Name)
+	case driver.SSH:
+		ip, err := host.Driver.GetIP()
+		if err != nil {
+			return []byte{}, errors.Wrap(err, "Error getting VM/Host IP address")
+		}
+		return net.ParseIP(ip), nil
 	case driver.KVM2:
 		return net.ParseIP("192.168.39.1"), nil
 	case driver.HyperV:

pkg/minikube/config/types.go

@@ -66,6 +66,10 @@ type ClusterConfig struct {
 	HostDNSResolver         bool   // Only used by virtualbox
 	HostOnlyNicType         string // Only used by virtualbox
 	NatNicType              string // Only used by virtualbox
+	IPAddress               string // Only used by ssh
+	SSHUser                 string // Only used by ssh
+	SSHKey                  string // Only used by ssh
+	SSHPort                 int    // Only used by ssh
 	KubernetesConfig        KubernetesConfig
 	Nodes                   []Node
 	Addons                  map[string]bool

pkg/minikube/driver/driver.go

@@ -38,6 +38,8 @@ const (
 	Mock = "mock"
 	// None driver
 	None = "none"
+	// SSH driver
+	SSH = "ssh"
 	// KVM2 driver
 	KVM2 = "kvm2"
 	// VirtualBox driver
@@ -55,6 +57,8 @@ const (
 
 	// AliasKVM is driver name alias for kvm2
 	AliasKVM = "kvm"
+	// AliasSSH is driver name alias for ssh
+	AliasSSH = "generic"
 )
 
 var (
@@ -96,6 +100,10 @@ func MachineType(name string) string {
 		return "container"
 	}
 
+	if IsSSH(name) {
+		return "bare metal machine"
+	}
+
 	if IsVM(name) {
 		return "VM"
 	}
@@ -143,6 +151,11 @@ func BareMetal(name string) bool {
 	return name == None || name == Mock
 }
 
+// IsSSH checks if the driver is ssh
+func IsSSH(name string) bool {
+	return name == SSH
+}
+
 // NeedsPortForward returns true if driver is unable provide direct IP connectivity
 func NeedsPortForward(name string) bool {
 	if !IsKIC(name) {

pkg/minikube/driver/driver_darwin.go

@@ -26,7 +26,7 @@ var supportedDrivers = []string{
 	HyperKit,
 	VMware,
 	Docker,
-	Podman,
+	SSH,
 }
 
 func VBoxManagePath() string {

pkg/minikube/driver/driver_linux.go

@@ -29,6 +29,7 @@ var supportedDrivers = []string{
 	None,
 	Docker,
 	Podman,
+	SSH,
 }
 
 // VBoxManagePath returns the path to the VBoxManage command

pkg/minikube/driver/driver_test.go

@@ -65,6 +65,7 @@ func TestMachineType(t *testing.T) {
 		Docker:       "container",
 		Mock:         "bare metal machine",
 		None:         "bare metal machine",
+		SSH:          "bare metal machine",
 		KVM2:         "VM",
 		VirtualBox:   "VM",
 		HyperKit:     "VM",