diff --git a/cmd/minikube/cmd/flags.go b/cmd/minikube/cmd/flags.go deleted file mode 100644 index 4c8c0efa0f11..000000000000 --- a/cmd/minikube/cmd/flags.go +++ /dev/null @@ -1,27 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "github.com/spf13/viper" - "k8s.io/minikube/pkg/minikube/config" -) - -// ClusterFlagValue returns the current cluster name based on flags -func ClusterFlagValue() string { - return viper.GetString(config.ProfileName) -} diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 0a42f5e979b1..72e7351bed25 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -27,7 +27,6 @@ import ( "os/user" "runtime" "strings" - "time" "github.com/blang/semver" "github.com/docker/machine/libmachine/ssh" @@ -44,11 +43,9 @@ import ( cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config" "k8s.io/minikube/pkg/drivers/kic/oci" "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil" - "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify" "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" - "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/exit" @@ -67,67 +64,6 @@ import ( "k8s.io/minikube/pkg/version" ) -const ( - isoURL = "iso-url" - memory = "memory" - cpus = "cpus" - humanReadableDiskSize = "disk-size" - nfsSharesRoot = "nfs-shares-root" - nfsShare = "nfs-share" - kubernetesVersion = "kubernetes-version" - hostOnlyCIDR = "host-only-cidr" - containerRuntime = "container-runtime" - criSocket = "cri-socket" - networkPlugin = "network-plugin" - enableDefaultCNI = "enable-default-cni" - hypervVirtualSwitch = "hyperv-virtual-switch" - hypervUseExternalSwitch = "hyperv-use-external-switch" - hypervExternalAdapter = "hyperv-external-adapter" - kvmNetwork = "kvm-network" - kvmQemuURI = "kvm-qemu-uri" - kvmGPU = "kvm-gpu" - kvmHidden = "kvm-hidden" - minikubeEnvPrefix = "MINIKUBE" - installAddons = "install-addons" - defaultDiskSize = "20000mb" - keepContext = "keep-context" - createMount = "mount" - featureGates = "feature-gates" - apiServerName = "apiserver-name" - apiServerPort = "apiserver-port" - dnsDomain = "dns-domain" - serviceCIDR = "service-cluster-ip-range" - imageRepository = "image-repository" - imageMirrorCountry = "image-mirror-country" - mountString = "mount-string" - disableDriverMounts = "disable-driver-mounts" - cacheImages = "cache-images" - uuid = "uuid" - vpnkitSock = "hyperkit-vpnkit-sock" - vsockPorts = "hyperkit-vsock-ports" - embedCerts = "embed-certs" - noVTXCheck = "no-vtx-check" - downloadOnly = "download-only" - dnsProxy = "dns-proxy" - hostDNSResolver = "host-dns-resolver" - waitComponents = "wait" - force = "force" - dryRun = "dry-run" - interactive = "interactive" - waitTimeout = "wait-timeout" - nativeSSH = "native-ssh" - minUsableMem = 1024 // Kubernetes will not start with less than 1GB - minRecommendedMem = 2000 // Warn at no lower than existing configurations - minimumCPUS = 2 - minimumDiskSize = 2000 - autoUpdate = "auto-update-drivers" - hostOnlyNicType = "host-only-nic-type" - natNicType = "nat-nic-type" - nodes = "nodes" - preload = "preload" - deleteOnFailure = "delete-on-failure" -) - var ( registryMirror []string insecureRegistry []string @@ -145,104 +81,6 @@ func init() { } } -// initMinikubeFlags includes commandline flags for minikube. -func initMinikubeFlags() { - viper.SetEnvPrefix(minikubeEnvPrefix) - // Replaces '-' in flags with '_' in env variables - // e.g. iso-url => $ENVPREFIX_ISO_URL - viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) - viper.AutomaticEnv() - - startCmd.Flags().Bool(force, false, "Force minikube to perform possibly dangerous operations") - startCmd.Flags().Bool(interactive, true, "Allow user prompts for more information") - startCmd.Flags().Bool(dryRun, false, "dry-run mode. Validates configuration, but does not mutate system state") - - startCmd.Flags().Int(cpus, 2, "Number of CPUs allocated to Kubernetes.") - startCmd.Flags().String(memory, "", "Amount of RAM to allocate to Kubernetes (format: [], where unit = b, k, m or g).") - startCmd.Flags().String(humanReadableDiskSize, defaultDiskSize, "Disk size allocated to the minikube VM (format: [], where unit = b, k, m or g).") - startCmd.Flags().Bool(downloadOnly, false, "If true, only download and cache files for later use - don't install or start anything.") - startCmd.Flags().Bool(cacheImages, true, "If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none.") - startCmd.Flags().StringSlice(isoURL, download.DefaultISOURLs(), "Locations to fetch the minikube ISO from.") - startCmd.Flags().Bool(keepContext, false, "This will keep the existing kubectl context and will create a minikube context.") - startCmd.Flags().Bool(embedCerts, false, "if true, will embed the certs in kubeconfig.") - startCmd.Flags().String(containerRuntime, "docker", "The container runtime to be used (docker, crio, containerd).") - startCmd.Flags().Bool(createMount, false, "This will start the mount daemon and automatically mount files into minikube.") - startCmd.Flags().String(mountString, constants.DefaultMountDir+":/minikube-host", "The argument to pass the minikube mount command on start.") - startCmd.Flags().StringArrayVar(&config.AddonList, "addons", nil, "Enable addons. see `minikube addons list` for a list of valid addon names.") - startCmd.Flags().String(criSocket, "", "The cri socket path to be used.") - startCmd.Flags().String(networkPlugin, "", "The name of the network plugin.") - startCmd.Flags().Bool(enableDefaultCNI, false, "Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \"--network-plugin=cni\".") - startCmd.Flags().StringSlice(waitComponents, kverify.DefaultWaitList, fmt.Sprintf("comma separated list of kubernetes components to verify and wait for after starting a cluster. defaults to %q, available options: %q . other acceptable values are 'all' or 'none', 'true' and 'false'", strings.Join(kverify.DefaultWaitList, ","), strings.Join(kverify.AllComponentsList, ","))) - startCmd.Flags().Duration(waitTimeout, 6*time.Minute, "max time to wait per Kubernetes core services to be healthy.") - startCmd.Flags().Bool(nativeSSH, true, "Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.") - startCmd.Flags().Bool(autoUpdate, true, "If set, automatically updates drivers to the latest version. Defaults to true.") - startCmd.Flags().Bool(installAddons, true, "If set, install addons. Defaults to true.") - startCmd.Flags().IntP(nodes, "n", 1, "The number of nodes to spin up. Defaults to 1.") - startCmd.Flags().Bool(preload, true, "If set, download tarball of preloaded images if available to improve start time. Defaults to true.") - startCmd.Flags().Bool(deleteOnFailure, false, "If set, delete the current cluster if start fails and try again. Defaults to false.") -} - -// initKubernetesFlags inits the commandline flags for kubernetes related options -func initKubernetesFlags() { - startCmd.Flags().String(kubernetesVersion, "", fmt.Sprintf("The kubernetes version that the minikube VM will use (ex: v1.2.3, 'stable' for %s, 'latest' for %s). Defaults to 'stable'.", constants.DefaultKubernetesVersion, constants.NewestKubernetesVersion)) - startCmd.Flags().Var(&config.ExtraOptions, "extra-config", - `A set of key=value pairs that describe configuration that may be passed to different components. - The key should be '.' separated, and the first part before the dot is the component to apply the configuration to. - Valid components are: kubelet, kubeadm, apiserver, controller-manager, etcd, proxy, scheduler - Valid kubeadm parameters: `+fmt.Sprintf("%s, %s", strings.Join(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmCmdParam], ", "), strings.Join(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmConfigParam], ","))) - startCmd.Flags().String(featureGates, "", "A set of key=value pairs that describe feature gates for alpha/experimental features.") - startCmd.Flags().String(dnsDomain, constants.ClusterDNSDomain, "The cluster dns domain name used in the kubernetes cluster") - startCmd.Flags().Int(apiServerPort, constants.APIServerPort, "The apiserver listening port") - startCmd.Flags().String(apiServerName, constants.APIServerName, "The authoritative apiserver hostname for apiserver certificates and connectivity. This can be used if you want to make the apiserver available from outside the machine") - startCmd.Flags().StringArrayVar(&apiServerNames, "apiserver-names", nil, "A set of apiserver names which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine") - startCmd.Flags().IPSliceVar(&apiServerIPs, "apiserver-ips", nil, "A set of apiserver IP Addresses which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine") -} - -// initDriverFlags inits the commandline flags for vm drivers -func initDriverFlags() { - startCmd.Flags().String("driver", "", fmt.Sprintf("Driver is one of: %v (defaults to auto-detect)", driver.DisplaySupportedDrivers())) - startCmd.Flags().String("vm-driver", "", "DEPRECATED, use `driver` instead.") - startCmd.Flags().Bool(disableDriverMounts, false, "Disables the filesystem mounts provided by the hypervisors") - startCmd.Flags().Bool("vm", false, "Filter to use only VM Drivers") - - // kvm2 - startCmd.Flags().String(kvmNetwork, "default", "The KVM network name. (kvm2 driver only)") - startCmd.Flags().String(kvmQemuURI, "qemu:///system", "The KVM QEMU connection URI. (kvm2 driver only)") - startCmd.Flags().Bool(kvmGPU, false, "Enable experimental NVIDIA GPU support in minikube") - startCmd.Flags().Bool(kvmHidden, false, "Hide the hypervisor signature from the guest in minikube (kvm2 driver only)") - - // virtualbox - startCmd.Flags().String(hostOnlyCIDR, "192.168.99.1/24", "The CIDR to be used for the minikube VM (virtualbox driver only)") - startCmd.Flags().Bool(dnsProxy, false, "Enable proxy for NAT DNS requests (virtualbox driver only)") - startCmd.Flags().Bool(hostDNSResolver, true, "Enable host resolver for NAT DNS requests (virtualbox driver only)") - startCmd.Flags().Bool(noVTXCheck, false, "Disable checking for the availability of hardware virtualization before the vm is started (virtualbox driver only)") - startCmd.Flags().String(hostOnlyNicType, "virtio", "NIC Type used for host only network. One of Am79C970A, Am79C973, 82540EM, 82543GC, 82545EM, or virtio (virtualbox driver only)") - startCmd.Flags().String(natNicType, "virtio", "NIC Type used for host only network. One of Am79C970A, Am79C973, 82540EM, 82543GC, 82545EM, or virtio (virtualbox driver only)") - - // hyperkit - startCmd.Flags().StringSlice(vsockPorts, []string{}, "List of guest VSock ports that should be exposed as sockets on the host (hyperkit driver only)") - startCmd.Flags().String(uuid, "", "Provide VM UUID to restore MAC address (hyperkit driver only)") - startCmd.Flags().String(vpnkitSock, "", "Location of the VPNKit socket used for networking. If empty, disables Hyperkit VPNKitSock, if 'auto' uses Docker for Mac VPNKit connection, otherwise uses the specified VSock (hyperkit driver only)") - startCmd.Flags().StringSlice(nfsShare, []string{}, "Local folders to share with Guest via NFS mounts (hyperkit driver only)") - startCmd.Flags().String(nfsSharesRoot, "/nfsshares", "Where to root the NFS Shares, defaults to /nfsshares (hyperkit driver only)") - - // hyperv - startCmd.Flags().String(hypervVirtualSwitch, "", "The hyperv virtual switch name. Defaults to first found. (hyperv driver only)") - startCmd.Flags().Bool(hypervUseExternalSwitch, false, "Whether to use external switch over Default Switch if virtual switch not explicitly specified. (hyperv driver only)") - startCmd.Flags().String(hypervExternalAdapter, "", "External Adapter on which external switch will be created if no external switch is found. (hyperv driver only)") -} - -// initNetworkingFlags inits the commandline flags for connectivity related flags for start -func initNetworkingFlags() { - startCmd.Flags().StringSliceVar(&insecureRegistry, "insecure-registry", nil, "Insecure Docker registries to pass to the Docker daemon. The default service CIDR range will automatically be added.") - startCmd.Flags().StringSliceVar(®istryMirror, "registry-mirror", nil, "Registry mirrors to pass to the Docker daemon") - startCmd.Flags().String(imageRepository, "", "Alternative image repository to pull docker images from. This can be used when you have limited access to gcr.io. Set it to \"auto\" to let minikube decide one for you. For Chinese mainland users, you may use local gcr.io mirrors such as registry.cn-hangzhou.aliyuncs.com/google_containers") - startCmd.Flags().String(imageMirrorCountry, "", "Country code of the image mirror to be used. Leave empty to use the global one. For Chinese mainland users, set it to cn.") - startCmd.Flags().String(serviceCIDR, constants.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.") - startCmd.Flags().StringArrayVar(&config.DockerEnv, "docker-env", nil, "Environment variables to pass to the Docker daemon. (format: key=value)") - startCmd.Flags().StringArrayVar(&config.DockerOpt, "docker-opt", nil, "Specify arbitrary flags to pass to the Docker daemon. (format: key=value)") -} - // startCmd represents the start command var startCmd = &cobra.Command{ Use: "start", @@ -323,7 +161,7 @@ func runStart(cmd *cobra.Command, args []string) { } k8sVersion := getKubernetesVersion(existing) - cc, n, err := generateCfgFromFlags(cmd, k8sVersion, driverName) + cc, n, err := generateClusterConfig(cmd, existing, k8sVersion, driverName) if err != nil { exit.WithError("Failed to generate config", err) } @@ -386,7 +224,7 @@ func runStart(cmd *cobra.Command, args []string) { } } - if err := showKubectlInfo(kubeconfig, k8sVersion, cc.Name); err != nil { + if err := showKubectlInfo(kubeconfig, cc.KubernetesConfig.KubernetesVersion, cc.Name); err != nil { glog.Errorf("kubectl info: %v", err) } } @@ -931,146 +769,17 @@ func validateRegistryMirror() { } } -// generateCfgFromFlags generates config.ClusterConfig based on flags and supplied arguments -func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string) (config.ClusterConfig, config.Node, error) { - r, err := cruntime.New(cruntime.Config{Type: viper.GetString(containerRuntime)}) - if err != nil { - return config.ClusterConfig{}, config.Node{}, err - } - - // Pick good default values for --network-plugin and --enable-default-cni based on runtime. - selectedEnableDefaultCNI := viper.GetBool(enableDefaultCNI) - selectedNetworkPlugin := viper.GetString(networkPlugin) - if r.DefaultCNI() && !cmd.Flags().Changed(networkPlugin) { - selectedNetworkPlugin = "cni" - if !cmd.Flags().Changed(enableDefaultCNI) { - selectedEnableDefaultCNI = true - } - } - - // Feed Docker our host proxy environment by default, so that it can pull images - if _, ok := r.(*cruntime.Docker); ok && !cmd.Flags().Changed("docker-env") { - setDockerProxy() - } - - repository := viper.GetString(imageRepository) - mirrorCountry := strings.ToLower(viper.GetString(imageMirrorCountry)) - if strings.ToLower(repository) == "auto" || mirrorCountry != "" { - found, autoSelectedRepository, err := selectImageRepository(mirrorCountry, semver.MustParse(strings.TrimPrefix(k8sVersion, version.VersionPrefix))) - if err != nil { - exit.WithError("Failed to check main repository and mirrors for images for images", err) - } - - if !found { - if autoSelectedRepository == "" { - exit.WithCodeT(exit.Failure, "None of the known repositories is accessible. Consider specifying an alternative image repository with --image-repository flag") - } else { - out.WarningT("None of the known repositories in your location are accessible. Using {{.image_repository_name}} as fallback.", out.V{"image_repository_name": autoSelectedRepository}) - } - } - - repository = autoSelectedRepository - } - - if cmd.Flags().Changed(imageRepository) { - out.T(out.SuccessType, "Using image repository {{.name}}", out.V{"name": repository}) - } - - var kubeNodeName string - if drvName != driver.None { - kubeNodeName = "m01" - } - - return createNode(cmd, k8sVersion, kubeNodeName, drvName, - repository, selectedEnableDefaultCNI, selectedNetworkPlugin) -} - -func createNode(cmd *cobra.Command, k8sVersion, kubeNodeName, drvName, repository string, - selectedEnableDefaultCNI bool, selectedNetworkPlugin string) (config.ClusterConfig, config.Node, error) { - - sysLimit, containerLimit, err := memoryLimits(drvName) - if err != nil { - glog.Warningf("Unable to query memory limits: %v", err) - } - - mem := suggestMemoryAllocation(sysLimit, containerLimit) - if cmd.Flags().Changed(memory) { - mem, err = pkgutil.CalculateSizeInMB(viper.GetString(memory)) - if err != nil { - exit.WithCodeT(exit.Config, "Generate unable to parse memory '{{.memory}}': {{.error}}", out.V{"memory": viper.GetString(memory), "error": err}) - } - - } else { - glog.Infof("Using suggested %dMB memory alloc based on sys=%dMB, container=%dMB", mem, sysLimit, containerLimit) - } - +func createNode(cc config.ClusterConfig, kubeNodeName string) (config.ClusterConfig, config.Node, error) { // Create the initial node, which will necessarily be a control plane cp := config.Node{ - Port: viper.GetInt(apiServerPort), - KubernetesVersion: k8sVersion, + Port: cc.KubernetesConfig.NodePort, + KubernetesVersion: getKubernetesVersion(&cc), Name: kubeNodeName, ControlPlane: true, Worker: true, } - - diskSize, err := pkgutil.CalculateSizeInMB(viper.GetString(humanReadableDiskSize)) - if err != nil { - exit.WithCodeT(exit.Config, "Generate unable to parse disk size '{{.diskSize}}': {{.error}}", out.V{"diskSize": viper.GetString(humanReadableDiskSize), "error": err}) - } - - cfg := config.ClusterConfig{ - Name: ClusterFlagValue(), - KeepContext: viper.GetBool(keepContext), - EmbedCerts: viper.GetBool(embedCerts), - MinikubeISO: viper.GetString(isoURL), - Memory: mem, - CPUs: viper.GetInt(cpus), - DiskSize: diskSize, - Driver: drvName, - HyperkitVpnKitSock: viper.GetString(vpnkitSock), - HyperkitVSockPorts: viper.GetStringSlice(vsockPorts), - NFSShare: viper.GetStringSlice(nfsShare), - NFSSharesRoot: viper.GetString(nfsSharesRoot), - DockerEnv: config.DockerEnv, - DockerOpt: config.DockerOpt, - InsecureRegistry: insecureRegistry, - RegistryMirror: registryMirror, - HostOnlyCIDR: viper.GetString(hostOnlyCIDR), - HypervVirtualSwitch: viper.GetString(hypervVirtualSwitch), - HypervUseExternalSwitch: viper.GetBool(hypervUseExternalSwitch), - HypervExternalAdapter: viper.GetString(hypervExternalAdapter), - KVMNetwork: viper.GetString(kvmNetwork), - KVMQemuURI: viper.GetString(kvmQemuURI), - KVMGPU: viper.GetBool(kvmGPU), - KVMHidden: viper.GetBool(kvmHidden), - DisableDriverMounts: viper.GetBool(disableDriverMounts), - UUID: viper.GetString(uuid), - NoVTXCheck: viper.GetBool(noVTXCheck), - DNSProxy: viper.GetBool(dnsProxy), - HostDNSResolver: viper.GetBool(hostDNSResolver), - HostOnlyNicType: viper.GetString(hostOnlyNicType), - NatNicType: viper.GetString(natNicType), - KubernetesConfig: config.KubernetesConfig{ - KubernetesVersion: k8sVersion, - ClusterName: ClusterFlagValue(), - APIServerName: viper.GetString(apiServerName), - APIServerNames: apiServerNames, - APIServerIPs: apiServerIPs, - DNSDomain: viper.GetString(dnsDomain), - FeatureGates: viper.GetString(featureGates), - ContainerRuntime: viper.GetString(containerRuntime), - CRISocket: viper.GetString(criSocket), - NetworkPlugin: selectedNetworkPlugin, - ServiceCIDR: viper.GetString(serviceCIDR), - ImageRepository: repository, - ExtraOptions: config.ExtraOptions, - ShouldLoadCachedImages: viper.GetBool(cacheImages), - EnableDefaultCNI: selectedEnableDefaultCNI, - }, - Nodes: []config.Node{cp}, - } - cfg.VerifyComponents = interpretWaitFlag(*cmd) - return cfg, cp, nil + cc.Nodes = []config.Node{cp} + return cc, cp, nil } // setDockerProxy sets the proxy environment variables in the docker environment. @@ -1202,48 +911,3 @@ func getKubernetesVersion(old *config.ClusterConfig) string { } return nv } - -// interpretWaitFlag interprets the wait flag and respects the legacy minikube users -// returns map of components to wait for -func interpretWaitFlag(cmd cobra.Command) map[string]bool { - if !cmd.Flags().Changed(waitComponents) { - glog.Infof("Wait components to verify : %+v", kverify.DefaultComponents) - return kverify.DefaultComponents - } - - waitFlags, err := cmd.Flags().GetStringSlice(waitComponents) - if err != nil { - glog.Warningf("Failed to read --wait from flags: %v.\n Moving on will use the default wait components: %+v", err, kverify.DefaultComponents) - return kverify.DefaultComponents - } - - if len(waitFlags) == 1 { - // respecting legacy flag before minikube 1.9.0, wait flag was boolean - if waitFlags[0] == "false" || waitFlags[0] == "none" { - glog.Infof("Waiting for no components: %+v", kverify.NoComponents) - return kverify.NoComponents - } - // respecting legacy flag before minikube 1.9.0, wait flag was boolean - if waitFlags[0] == "true" || waitFlags[0] == "all" { - glog.Infof("Waiting for all components: %+v", kverify.AllComponents) - return kverify.AllComponents - } - } - - waitComponents := kverify.NoComponents - for _, wc := range waitFlags { - seen := false - for _, valid := range kverify.AllComponentsList { - if wc == valid { - waitComponents[wc] = true - seen = true - continue - } - } - if !seen { - glog.Warningf("The value %q is invalid for --wait flag. valid options are %q", wc, strings.Join(kverify.AllComponentsList, ",")) - } - } - glog.Infof("Waiting for components: %+v", waitComponents) - return waitComponents -} diff --git a/cmd/minikube/cmd/start_flags.go b/cmd/minikube/cmd/start_flags.go new file mode 100644 index 000000000000..b0556c79a2b7 --- /dev/null +++ b/cmd/minikube/cmd/start_flags.go @@ -0,0 +1,587 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "fmt" + "strings" + "time" + + "github.com/blang/semver" + "github.com/golang/glog" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil" + "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/cruntime" + "k8s.io/minikube/pkg/minikube/download" + "k8s.io/minikube/pkg/minikube/driver" + "k8s.io/minikube/pkg/minikube/exit" + "k8s.io/minikube/pkg/minikube/out" + pkgutil "k8s.io/minikube/pkg/util" + "k8s.io/minikube/pkg/version" +) + +const ( + isoURL = "iso-url" + memory = "memory" + cpus = "cpus" + humanReadableDiskSize = "disk-size" + nfsSharesRoot = "nfs-shares-root" + nfsShare = "nfs-share" + kubernetesVersion = "kubernetes-version" + hostOnlyCIDR = "host-only-cidr" + containerRuntime = "container-runtime" + criSocket = "cri-socket" + networkPlugin = "network-plugin" + enableDefaultCNI = "enable-default-cni" + hypervVirtualSwitch = "hyperv-virtual-switch" + hypervUseExternalSwitch = "hyperv-use-external-switch" + hypervExternalAdapter = "hyperv-external-adapter" + kvmNetwork = "kvm-network" + kvmQemuURI = "kvm-qemu-uri" + kvmGPU = "kvm-gpu" + kvmHidden = "kvm-hidden" + minikubeEnvPrefix = "MINIKUBE" + installAddons = "install-addons" + defaultDiskSize = "20000mb" + keepContext = "keep-context" + createMount = "mount" + featureGates = "feature-gates" + apiServerName = "apiserver-name" + apiServerPort = "apiserver-port" + dnsDomain = "dns-domain" + serviceCIDR = "service-cluster-ip-range" + imageRepository = "image-repository" + imageMirrorCountry = "image-mirror-country" + mountString = "mount-string" + disableDriverMounts = "disable-driver-mounts" + cacheImages = "cache-images" + uuid = "uuid" + vpnkitSock = "hyperkit-vpnkit-sock" + vsockPorts = "hyperkit-vsock-ports" + embedCerts = "embed-certs" + noVTXCheck = "no-vtx-check" + downloadOnly = "download-only" + dnsProxy = "dns-proxy" + hostDNSResolver = "host-dns-resolver" + waitComponents = "wait" + force = "force" + dryRun = "dry-run" + interactive = "interactive" + waitTimeout = "wait-timeout" + nativeSSH = "native-ssh" + minUsableMem = 1024 // Kubernetes will not start with less than 1GB + minRecommendedMem = 2000 // Warn at no lower than existing configurations + minimumCPUS = 2 + minimumDiskSize = 2000 + autoUpdate = "auto-update-drivers" + hostOnlyNicType = "host-only-nic-type" + natNicType = "nat-nic-type" + nodes = "nodes" + preload = "preload" + deleteOnFailure = "delete-on-failure" +) + +// initMinikubeFlags includes commandline flags for minikube. +func initMinikubeFlags() { + viper.SetEnvPrefix(minikubeEnvPrefix) + // Replaces '-' in flags with '_' in env variables + // e.g. iso-url => $ENVPREFIX_ISO_URL + viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + viper.AutomaticEnv() + + startCmd.Flags().Bool(force, false, "Force minikube to perform possibly dangerous operations") + startCmd.Flags().Bool(interactive, true, "Allow user prompts for more information") + startCmd.Flags().Bool(dryRun, false, "dry-run mode. Validates configuration, but does not mutate system state") + + startCmd.Flags().Int(cpus, 2, "Number of CPUs allocated to Kubernetes.") + startCmd.Flags().String(memory, "", "Amount of RAM to allocate to Kubernetes (format: [], where unit = b, k, m or g).") + startCmd.Flags().String(humanReadableDiskSize, defaultDiskSize, "Disk size allocated to the minikube VM (format: [], where unit = b, k, m or g).") + startCmd.Flags().Bool(downloadOnly, false, "If true, only download and cache files for later use - don't install or start anything.") + startCmd.Flags().Bool(cacheImages, true, "If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none.") + startCmd.Flags().StringSlice(isoURL, download.DefaultISOURLs(), "Locations to fetch the minikube ISO from.") + startCmd.Flags().Bool(keepContext, false, "This will keep the existing kubectl context and will create a minikube context.") + startCmd.Flags().Bool(embedCerts, false, "if true, will embed the certs in kubeconfig.") + startCmd.Flags().String(containerRuntime, "docker", "The container runtime to be used (docker, crio, containerd).") + startCmd.Flags().Bool(createMount, false, "This will start the mount daemon and automatically mount files into minikube.") + startCmd.Flags().String(mountString, constants.DefaultMountDir+":/minikube-host", "The argument to pass the minikube mount command on start.") + startCmd.Flags().StringArrayVar(&config.AddonList, "addons", nil, "Enable addons. see `minikube addons list` for a list of valid addon names.") + startCmd.Flags().String(criSocket, "", "The cri socket path to be used.") + startCmd.Flags().String(networkPlugin, "", "The name of the network plugin.") + startCmd.Flags().Bool(enableDefaultCNI, false, "Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \"--network-plugin=cni\".") + startCmd.Flags().StringSlice(waitComponents, kverify.DefaultWaitList, fmt.Sprintf("comma separated list of kubernetes components to verify and wait for after starting a cluster. defaults to %q, available options: %q . other acceptable values are 'all' or 'none', 'true' and 'false'", strings.Join(kverify.DefaultWaitList, ","), strings.Join(kverify.AllComponentsList, ","))) + startCmd.Flags().Duration(waitTimeout, 6*time.Minute, "max time to wait per Kubernetes core services to be healthy.") + startCmd.Flags().Bool(nativeSSH, true, "Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.") + startCmd.Flags().Bool(autoUpdate, true, "If set, automatically updates drivers to the latest version. Defaults to true.") + startCmd.Flags().Bool(installAddons, true, "If set, install addons. Defaults to true.") + startCmd.Flags().IntP(nodes, "n", 1, "The number of nodes to spin up. Defaults to 1.") + startCmd.Flags().Bool(preload, true, "If set, download tarball of preloaded images if available to improve start time. Defaults to true.") + startCmd.Flags().Bool(deleteOnFailure, false, "If set, delete the current cluster if start fails and try again. Defaults to false.") +} + +// initKubernetesFlags inits the commandline flags for kubernetes related options +func initKubernetesFlags() { + startCmd.Flags().String(kubernetesVersion, "", fmt.Sprintf("The kubernetes version that the minikube VM will use (ex: v1.2.3, 'stable' for %s, 'latest' for %s). Defaults to 'stable'.", constants.DefaultKubernetesVersion, constants.NewestKubernetesVersion)) + startCmd.Flags().Var(&config.ExtraOptions, "extra-config", + `A set of key=value pairs that describe configuration that may be passed to different components. + The key should be '.' separated, and the first part before the dot is the component to apply the configuration to. + Valid components are: kubelet, kubeadm, apiserver, controller-manager, etcd, proxy, scheduler + Valid kubeadm parameters: `+fmt.Sprintf("%s, %s", strings.Join(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmCmdParam], ", "), strings.Join(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmConfigParam], ","))) + startCmd.Flags().String(featureGates, "", "A set of key=value pairs that describe feature gates for alpha/experimental features.") + startCmd.Flags().String(dnsDomain, constants.ClusterDNSDomain, "The cluster dns domain name used in the kubernetes cluster") + startCmd.Flags().Int(apiServerPort, constants.APIServerPort, "The apiserver listening port") + startCmd.Flags().String(apiServerName, constants.APIServerName, "The authoritative apiserver hostname for apiserver certificates and connectivity. This can be used if you want to make the apiserver available from outside the machine") + startCmd.Flags().StringArrayVar(&apiServerNames, "apiserver-names", nil, "A set of apiserver names which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine") + startCmd.Flags().IPSliceVar(&apiServerIPs, "apiserver-ips", nil, "A set of apiserver IP Addresses which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine") +} + +// initDriverFlags inits the commandline flags for vm drivers +func initDriverFlags() { + startCmd.Flags().String("driver", "", fmt.Sprintf("Driver is one of: %v (defaults to auto-detect)", driver.DisplaySupportedDrivers())) + startCmd.Flags().String("vm-driver", "", "DEPRECATED, use `driver` instead.") + startCmd.Flags().Bool(disableDriverMounts, false, "Disables the filesystem mounts provided by the hypervisors") + startCmd.Flags().Bool("vm", false, "Filter to use only VM Drivers") + + // kvm2 + startCmd.Flags().String(kvmNetwork, "default", "The KVM network name. (kvm2 driver only)") + startCmd.Flags().String(kvmQemuURI, "qemu:///system", "The KVM QEMU connection URI. (kvm2 driver only)") + startCmd.Flags().Bool(kvmGPU, false, "Enable experimental NVIDIA GPU support in minikube") + startCmd.Flags().Bool(kvmHidden, false, "Hide the hypervisor signature from the guest in minikube (kvm2 driver only)") + + // virtualbox + startCmd.Flags().String(hostOnlyCIDR, "192.168.99.1/24", "The CIDR to be used for the minikube VM (virtualbox driver only)") + startCmd.Flags().Bool(dnsProxy, false, "Enable proxy for NAT DNS requests (virtualbox driver only)") + startCmd.Flags().Bool(hostDNSResolver, true, "Enable host resolver for NAT DNS requests (virtualbox driver only)") + startCmd.Flags().Bool(noVTXCheck, false, "Disable checking for the availability of hardware virtualization before the vm is started (virtualbox driver only)") + startCmd.Flags().String(hostOnlyNicType, "virtio", "NIC Type used for host only network. One of Am79C970A, Am79C973, 82540EM, 82543GC, 82545EM, or virtio (virtualbox driver only)") + startCmd.Flags().String(natNicType, "virtio", "NIC Type used for host only network. One of Am79C970A, Am79C973, 82540EM, 82543GC, 82545EM, or virtio (virtualbox driver only)") + + // hyperkit + startCmd.Flags().StringSlice(vsockPorts, []string{}, "List of guest VSock ports that should be exposed as sockets on the host (hyperkit driver only)") + startCmd.Flags().String(uuid, "", "Provide VM UUID to restore MAC address (hyperkit driver only)") + startCmd.Flags().String(vpnkitSock, "", "Location of the VPNKit socket used for networking. If empty, disables Hyperkit VPNKitSock, if 'auto' uses Docker for Mac VPNKit connection, otherwise uses the specified VSock (hyperkit driver only)") + startCmd.Flags().StringSlice(nfsShare, []string{}, "Local folders to share with Guest via NFS mounts (hyperkit driver only)") + startCmd.Flags().String(nfsSharesRoot, "/nfsshares", "Where to root the NFS Shares, defaults to /nfsshares (hyperkit driver only)") + + // hyperv + startCmd.Flags().String(hypervVirtualSwitch, "", "The hyperv virtual switch name. Defaults to first found. (hyperv driver only)") + startCmd.Flags().Bool(hypervUseExternalSwitch, false, "Whether to use external switch over Default Switch if virtual switch not explicitly specified. (hyperv driver only)") + startCmd.Flags().String(hypervExternalAdapter, "", "External Adapter on which external switch will be created if no external switch is found. (hyperv driver only)") +} + +// initNetworkingFlags inits the commandline flags for connectivity related flags for start +func initNetworkingFlags() { + startCmd.Flags().StringSliceVar(&insecureRegistry, "insecure-registry", nil, "Insecure Docker registries to pass to the Docker daemon. The default service CIDR range will automatically be added.") + startCmd.Flags().StringSliceVar(®istryMirror, "registry-mirror", nil, "Registry mirrors to pass to the Docker daemon") + startCmd.Flags().String(imageRepository, "", "Alternative image repository to pull docker images from. This can be used when you have limited access to gcr.io. Set it to \"auto\" to let minikube decide one for you. For Chinese mainland users, you may use local gcr.io mirrors such as registry.cn-hangzhou.aliyuncs.com/google_containers") + startCmd.Flags().String(imageMirrorCountry, "", "Country code of the image mirror to be used. Leave empty to use the global one. For Chinese mainland users, set it to cn.") + startCmd.Flags().String(serviceCIDR, constants.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.") + startCmd.Flags().StringArrayVar(&config.DockerEnv, "docker-env", nil, "Environment variables to pass to the Docker daemon. (format: key=value)") + startCmd.Flags().StringArrayVar(&config.DockerOpt, "docker-opt", nil, "Specify arbitrary flags to pass to the Docker daemon. (format: key=value)") +} + +// ClusterFlagValue returns the current cluster name based on flags +func ClusterFlagValue() string { + return viper.GetString(config.ProfileName) +} + +// generateClusterConfig generate a config.ClusterConfig based on flags or existing cluster config +func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k8sVersion string, drvName string) (config.ClusterConfig, config.Node, error) { + cc := config.ClusterConfig{} + if existing != nil { // create profile config first time + cc = updateExistingConfigFromFlags(cmd, existing) + } else { + glog.Info("no existing cluster config was found, will generate one from the flags ") + sysLimit, containerLimit, err := memoryLimits(drvName) + if err != nil { + glog.Warningf("Unable to query memory limits: %v", err) + } + + mem := suggestMemoryAllocation(sysLimit, containerLimit) + if cmd.Flags().Changed(memory) { + mem, err = pkgutil.CalculateSizeInMB(viper.GetString(memory)) + if err != nil { + exit.WithCodeT(exit.Config, "Generate unable to parse memory '{{.memory}}': {{.error}}", out.V{"memory": viper.GetString(memory), "error": err}) + } + + } else { + glog.Infof("Using suggested %dMB memory alloc based on sys=%dMB, container=%dMB", mem, sysLimit, containerLimit) + } + + diskSize, err := pkgutil.CalculateSizeInMB(viper.GetString(humanReadableDiskSize)) + if err != nil { + exit.WithCodeT(exit.Config, "Generate unable to parse disk size '{{.diskSize}}': {{.error}}", out.V{"diskSize": viper.GetString(humanReadableDiskSize), "error": err}) + } + + r, err := cruntime.New(cruntime.Config{Type: viper.GetString(containerRuntime)}) + if err != nil { + return cc, config.Node{}, errors.Wrap(err, "new runtime manager") + } + + if cmd.Flags().Changed(imageRepository) { + cc.KubernetesConfig.ImageRepository = viper.GetString(imageRepository) + } + + // Pick good default values for --network-plugin and --enable-default-cni based on runtime. + selectedEnableDefaultCNI := viper.GetBool(enableDefaultCNI) + selectedNetworkPlugin := viper.GetString(networkPlugin) + if r.DefaultCNI() && !cmd.Flags().Changed(networkPlugin) { + selectedNetworkPlugin = "cni" + if !cmd.Flags().Changed(enableDefaultCNI) { + selectedEnableDefaultCNI = true + } + } + + repository := viper.GetString(imageRepository) + mirrorCountry := strings.ToLower(viper.GetString(imageMirrorCountry)) + if strings.ToLower(repository) == "auto" || mirrorCountry != "" { + found, autoSelectedRepository, err := selectImageRepository(mirrorCountry, semver.MustParse(strings.TrimPrefix(k8sVersion, version.VersionPrefix))) + if err != nil { + exit.WithError("Failed to check main repository and mirrors for images for images", err) + } + + if !found { + if autoSelectedRepository == "" { + exit.WithCodeT(exit.Failure, "None of the known repositories is accessible. Consider specifying an alternative image repository with --image-repository flag") + } else { + out.WarningT("None of the known repositories in your location are accessible. Using {{.image_repository_name}} as fallback.", out.V{"image_repository_name": autoSelectedRepository}) + } + } + + repository = autoSelectedRepository + } + + if cmd.Flags().Changed(imageRepository) { + out.T(out.SuccessType, "Using image repository {{.name}}", out.V{"name": repository}) + } + + cc = config.ClusterConfig{ + Name: ClusterFlagValue(), + KeepContext: viper.GetBool(keepContext), + EmbedCerts: viper.GetBool(embedCerts), + MinikubeISO: viper.GetString(isoURL), + Memory: mem, + CPUs: viper.GetInt(cpus), + DiskSize: diskSize, + Driver: drvName, + HyperkitVpnKitSock: viper.GetString(vpnkitSock), + HyperkitVSockPorts: viper.GetStringSlice(vsockPorts), + NFSShare: viper.GetStringSlice(nfsShare), + NFSSharesRoot: viper.GetString(nfsSharesRoot), + DockerEnv: config.DockerEnv, + DockerOpt: config.DockerOpt, + InsecureRegistry: insecureRegistry, + RegistryMirror: registryMirror, + HostOnlyCIDR: viper.GetString(hostOnlyCIDR), + HypervVirtualSwitch: viper.GetString(hypervVirtualSwitch), + HypervUseExternalSwitch: viper.GetBool(hypervUseExternalSwitch), + HypervExternalAdapter: viper.GetString(hypervExternalAdapter), + KVMNetwork: viper.GetString(kvmNetwork), + KVMQemuURI: viper.GetString(kvmQemuURI), + KVMGPU: viper.GetBool(kvmGPU), + KVMHidden: viper.GetBool(kvmHidden), + DisableDriverMounts: viper.GetBool(disableDriverMounts), + UUID: viper.GetString(uuid), + NoVTXCheck: viper.GetBool(noVTXCheck), + DNSProxy: viper.GetBool(dnsProxy), + HostDNSResolver: viper.GetBool(hostDNSResolver), + HostOnlyNicType: viper.GetString(hostOnlyNicType), + NatNicType: viper.GetString(natNicType), + KubernetesConfig: config.KubernetesConfig{ + KubernetesVersion: k8sVersion, + ClusterName: ClusterFlagValue(), + APIServerName: viper.GetString(apiServerName), + APIServerNames: apiServerNames, + APIServerIPs: apiServerIPs, + DNSDomain: viper.GetString(dnsDomain), + FeatureGates: viper.GetString(featureGates), + ContainerRuntime: viper.GetString(containerRuntime), + CRISocket: viper.GetString(criSocket), + NetworkPlugin: selectedNetworkPlugin, + ServiceCIDR: viper.GetString(serviceCIDR), + ImageRepository: repository, + ExtraOptions: config.ExtraOptions, + ShouldLoadCachedImages: viper.GetBool(cacheImages), + EnableDefaultCNI: selectedEnableDefaultCNI, + NodePort: viper.GetInt(apiServerPort), + }, + } + cc.VerifyComponents = interpretWaitFlag(*cmd) + } + + r, err := cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime}) + if err != nil { + return cc, config.Node{}, errors.Wrap(err, "new runtime manager") + } + + // Feed Docker our host proxy environment by default, so that it can pull images + // doing this for both new config and existing, in case proxy changed since previous start + if _, ok := r.(*cruntime.Docker); ok && !cmd.Flags().Changed("docker-env") { + setDockerProxy() + } + + var kubeNodeName string + if driver.BareMetal(cc.Driver) { + kubeNodeName = "m01" + } + return createNode(cc, kubeNodeName) +} + +// updateExistingConfigFromFlags will update the existing config from the flags - used on a second start +// skipping updating existing docker env , docker opt, InsecureRegistry, registryMirror, extra-config, apiserver-ips +func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterConfig) config.ClusterConfig { //nolint to supress cyclomatic complexity 45 of func `updateExistingConfigFromFlags` is high (> 30) + validateFlags(cmd, existing.Driver) + + if cmd.Flags().Changed(containerRuntime) { + existing.KubernetesConfig.ContainerRuntime = viper.GetString(containerRuntime) + } + + if cmd.Flags().Changed(keepContext) { + existing.KeepContext = viper.GetBool(keepContext) + } + + if cmd.Flags().Changed(embedCerts) { + existing.EmbedCerts = viper.GetBool(embedCerts) + } + + if cmd.Flags().Changed(isoURL) { + existing.MinikubeISO = viper.GetString(isoURL) + } + + if cmd.Flags().Changed(memory) { + memInMB, err := pkgutil.CalculateSizeInMB(viper.GetString(memory)) + if err != nil { + glog.Warningf("error calculate memory size in mb : %v", err) + } + if memInMB != existing.Memory { + out.WarningT("You not the change the memory size for an exiting minikube cluster. Pease first delete the cluster.") + } + + } + + if cmd.Flags().Changed(cpus) { + if viper.GetInt(cpus) != existing.CPUs { + out.WarningT("You not the change the CPUs for an exiting minikube cluster. Pease first delete the cluster.") + } + } + + if cmd.Flags().Changed(humanReadableDiskSize) { + memInMB, err := pkgutil.CalculateSizeInMB(viper.GetString(humanReadableDiskSize)) + if err != nil { + glog.Warningf("error calculate disk size in mb : %v", err) + } + + if memInMB != existing.DiskSize { + out.WarningT("You not the change the Disk size for an exiting minikube cluster. Pease first delete the cluster.") + } + } + + if cmd.Flags().Changed(vpnkitSock) { + existing.HyperkitVpnKitSock = viper.GetString(vpnkitSock) + } + + if cmd.Flags().Changed(vsockPorts) { + existing.HyperkitVSockPorts = viper.GetStringSlice(vsockPorts) + } + + if cmd.Flags().Changed(nfsShare) { + existing.NFSShare = viper.GetStringSlice(nfsShare) + } + + if cmd.Flags().Changed(nfsSharesRoot) { + existing.NFSSharesRoot = viper.GetString(nfsSharesRoot) + } + + if cmd.Flags().Changed(hostOnlyCIDR) { + existing.HostOnlyCIDR = viper.GetString(hostOnlyCIDR) + } + + if cmd.Flags().Changed(hypervVirtualSwitch) { + existing.HypervVirtualSwitch = viper.GetString(hypervVirtualSwitch) + } + + if cmd.Flags().Changed(hypervUseExternalSwitch) { + existing.HypervUseExternalSwitch = viper.GetBool(hypervUseExternalSwitch) + } + + if cmd.Flags().Changed(hypervExternalAdapter) { + existing.HypervExternalAdapter = viper.GetString(hypervExternalAdapter) + } + + if cmd.Flags().Changed(kvmNetwork) { + existing.KVMNetwork = viper.GetString(kvmNetwork) + } + + if cmd.Flags().Changed(kvmQemuURI) { + existing.KVMQemuURI = viper.GetString(kvmQemuURI) + } + + if cmd.Flags().Changed(kvmGPU) { + existing.KVMGPU = viper.GetBool(kvmGPU) + } + + if cmd.Flags().Changed(kvmHidden) { + existing.KVMHidden = viper.GetBool(kvmHidden) + } + + if cmd.Flags().Changed(disableDriverMounts) { + existing.DisableDriverMounts = viper.GetBool(disableDriverMounts) + } + + if cmd.Flags().Changed(uuid) { + existing.UUID = viper.GetString(uuid) + } + + if cmd.Flags().Changed(noVTXCheck) { + existing.NoVTXCheck = viper.GetBool(noVTXCheck) + } + + if cmd.Flags().Changed(dnsProxy) { + existing.DNSProxy = viper.GetBool(dnsProxy) + } + + if cmd.Flags().Changed(hostDNSResolver) { + existing.HostDNSResolver = viper.GetBool(hostDNSResolver) + } + + if cmd.Flags().Changed(hostOnlyNicType) { + existing.HostOnlyNicType = viper.GetString(hostOnlyNicType) + } + + if cmd.Flags().Changed(natNicType) { + existing.NatNicType = viper.GetString(natNicType) + } + + if cmd.Flags().Changed(kubernetesVersion) { + existing.KubernetesConfig.KubernetesVersion = viper.GetString(kubernetesVersion) + } + + if cmd.Flags().Changed(apiServerName) { + existing.KubernetesConfig.APIServerName = viper.GetString(apiServerName) + } + + if cmd.Flags().Changed("apiserver-names") { + existing.KubernetesConfig.APIServerNames = viper.GetStringSlice("apiserver-names") + } + + if cmd.Flags().Changed(apiServerPort) { + existing.KubernetesConfig.NodePort = viper.GetInt(apiServerPort) + } + + // pre minikube 1.9.2 cc.KubernetesConfig.NodePort was not populated. + // in minikube config there were two fields for api server port. + // one in cc.KubernetesConfig.NodePort and one in cc.Nodes.Port + // this makes sure api server port not be set as 0! + if existing.KubernetesConfig.NodePort == 0 { + existing.KubernetesConfig.NodePort = viper.GetInt(apiServerPort) + } + + if cmd.Flags().Changed(dnsDomain) { + existing.KubernetesConfig.DNSDomain = viper.GetString(dnsDomain) + } + + if cmd.Flags().Changed(featureGates) { + existing.KubernetesConfig.FeatureGates = viper.GetString(featureGates) + } + + if cmd.Flags().Changed(containerRuntime) { + existing.KubernetesConfig.ContainerRuntime = viper.GetString(containerRuntime) + } + + if cmd.Flags().Changed(criSocket) { + existing.KubernetesConfig.CRISocket = viper.GetString(criSocket) + } + + if cmd.Flags().Changed(criSocket) { + existing.KubernetesConfig.NetworkPlugin = viper.GetString(criSocket) + } + + if cmd.Flags().Changed(networkPlugin) { + existing.KubernetesConfig.NetworkPlugin = viper.GetString(networkPlugin) + } + + if cmd.Flags().Changed(serviceCIDR) { + existing.KubernetesConfig.ServiceCIDR = viper.GetString(serviceCIDR) + } + + if cmd.Flags().Changed(cacheImages) { + existing.KubernetesConfig.ShouldLoadCachedImages = viper.GetBool(cacheImages) + } + + if cmd.Flags().Changed(imageRepository) { + existing.KubernetesConfig.ImageRepository = viper.GetString(imageRepository) + } + + if cmd.Flags().Changed(enableDefaultCNI) { + existing.KubernetesConfig.EnableDefaultCNI = viper.GetBool(enableDefaultCNI) + } + + if cmd.Flags().Changed(waitComponents) { + existing.VerifyComponents = interpretWaitFlag(*cmd) + } + + return *existing +} + +// interpretWaitFlag interprets the wait flag and respects the legacy minikube users +// returns map of components to wait for +func interpretWaitFlag(cmd cobra.Command) map[string]bool { + if !cmd.Flags().Changed(waitComponents) { + glog.Infof("Wait components to verify : %+v", kverify.DefaultComponents) + return kverify.DefaultComponents + } + + waitFlags, err := cmd.Flags().GetStringSlice(waitComponents) + if err != nil { + glog.Warningf("Failed to read --wait from flags: %v.\n Moving on will use the default wait components: %+v", err, kverify.DefaultComponents) + return kverify.DefaultComponents + } + + if len(waitFlags) == 1 { + // respecting legacy flag before minikube 1.9.0, wait flag was boolean + if waitFlags[0] == "false" || waitFlags[0] == "none" { + glog.Infof("Waiting for no components: %+v", kverify.NoComponents) + return kverify.NoComponents + } + // respecting legacy flag before minikube 1.9.0, wait flag was boolean + if waitFlags[0] == "true" || waitFlags[0] == "all" { + glog.Infof("Waiting for all components: %+v", kverify.AllComponents) + return kverify.AllComponents + } + } + + waitComponents := kverify.NoComponents + for _, wc := range waitFlags { + seen := false + for _, valid := range kverify.AllComponentsList { + if wc == valid { + waitComponents[wc] = true + seen = true + continue + } + } + if !seen { + glog.Warningf("The value %q is invalid for --wait flag. valid options are %q", wc, strings.Join(kverify.AllComponentsList, ",")) + } + } + glog.Infof("Waiting for components: %+v", waitComponents) + return waitComponents +} diff --git a/cmd/minikube/cmd/start_test.go b/cmd/minikube/cmd/start_test.go index 0a443aa50793..9c6dfa93a1af 100644 --- a/cmd/minikube/cmd/start_test.go +++ b/cmd/minikube/cmd/start_test.go @@ -114,7 +114,7 @@ func TestMirrorCountry(t *testing.T) { cmd := &cobra.Command{} viper.SetDefault(imageRepository, test.imageRepository) viper.SetDefault(imageMirrorCountry, test.mirrorCountry) - config, _, err := generateCfgFromFlags(cmd, k8sVersion, "none") + config, _, err := generateClusterConfig(cmd, nil, k8sVersion, "none") if err != nil { t.Fatalf("Got unexpected error %v during config generation", err) } @@ -166,7 +166,7 @@ func TestGenerateCfgFromFlagsHTTPProxyHandling(t *testing.T) { if err := os.Setenv("HTTP_PROXY", test.proxy); err != nil { t.Fatalf("Unexpected error setting HTTP_PROXY: %v", err) } - config, _, err := generateCfgFromFlags(cmd, k8sVersion, "none") + config, _, err := generateClusterConfig(cmd, nil, k8sVersion, "none") if err != nil { t.Fatalf("Got unexpected error %v during config generation", err) } diff --git a/pkg/minikube/config/config.go b/pkg/minikube/config/config.go index f11816070c01..4db4af8dc56d 100644 --- a/pkg/minikube/config/config.go +++ b/pkg/minikube/config/config.go @@ -193,7 +193,6 @@ func (c *simpleConfigLoader) LoadConfigFromFile(profileName string, miniHome ... } func (c *simpleConfigLoader) WriteConfigToFile(profileName string, cc *ClusterConfig, miniHome ...string) error { - // Move to profile package path := profileFilePath(profileName, miniHome...) contents, err := json.MarshalIndent(cc, "", " ") if err != nil { diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 9f4dbbd0bf59..28cac0476c75 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -21,6 +21,7 @@ import ( "net" "os" "os/exec" + "runtime/debug" "strconv" "strings" "sync" @@ -56,13 +57,7 @@ import ( "k8s.io/minikube/pkg/util/retry" ) -const ( - waitTimeout = "wait-timeout" - embedCerts = "embed-certs" - keepContext = "keep-context" - imageRepository = "image-repository" - containerRuntime = "container-runtime" -) +const waitTimeout = "wait-timeout" // Start spins up a guest and starts the kubernetes node. func Start(cc config.ClusterConfig, n config.Node, existingAddons map[string]bool, apiServer bool) (*kubeconfig.Settings, error) { @@ -103,7 +98,7 @@ func Start(cc config.ClusterConfig, n config.Node, existingAddons map[string]boo } // configure the runtime (docker, containerd, crio) - cr := configureRuntimes(mRunner, cc.Driver, cc.KubernetesConfig, sv) + cr := configureRuntimes(mRunner, cc, sv) showVersionInfo(n.KubernetesVersion, cr) var bs bootstrapper.Bootstrapper @@ -189,10 +184,11 @@ func Start(cc config.ClusterConfig, n config.Node, existingAddons map[string]boo } // ConfigureRuntimes does what needs to happen to get a runtime going. -func configureRuntimes(runner cruntime.CommandRunner, drvName string, k8s config.KubernetesConfig, kv semver.Version) cruntime.Manager { +func configureRuntimes(runner cruntime.CommandRunner, cc config.ClusterConfig, kv semver.Version) cruntime.Manager { co := cruntime.Config{ - Type: viper.GetString(containerRuntime), - Runner: runner, ImageRepository: k8s.ImageRepository, + Type: cc.KubernetesConfig.ContainerRuntime, + Runner: runner, + ImageRepository: cc.KubernetesConfig.ImageRepository, KubernetesVersion: kv, } cr, err := cruntime.New(co) @@ -201,13 +197,13 @@ func configureRuntimes(runner cruntime.CommandRunner, drvName string, k8s config } disableOthers := true - if driver.BareMetal(drvName) { + if driver.BareMetal(cc.Driver) { disableOthers = false } // Preload is overly invasive for bare metal, and caching is not meaningful. KIC handled elsewhere. - if driver.IsVM(drvName) { - if err := cr.Preload(k8s); err != nil { + if driver.IsVM(cc.Driver) { + if err := cr.Preload(cc.KubernetesConfig); err != nil { switch err.(type) { case *cruntime.ErrISOFeature: out.ErrT(out.Tip, "Existing disk is missing new features ({{.error}}). To upgrade, run 'minikube delete'", out.V{"error": err}) @@ -215,7 +211,7 @@ func configureRuntimes(runner cruntime.CommandRunner, drvName string, k8s config glog.Warningf("%s preload failed: %v, falling back to caching images", cr.Name(), err) } - if err := machine.CacheImagesForBootstrapper(k8s.ImageRepository, k8s.KubernetesVersion, viper.GetString(cmdcfg.Bootstrapper)); err != nil { + if err := machine.CacheImagesForBootstrapper(cc.KubernetesConfig.ImageRepository, cc.KubernetesConfig.KubernetesVersion, viper.GetString(cmdcfg.Bootstrapper)); err != nil { exit.WithError("Failed to cache images", err) } } @@ -223,6 +219,7 @@ func configureRuntimes(runner cruntime.CommandRunner, drvName string, k8s config err = cr.Enable(disableOthers) if err != nil { + debug.PrintStack() exit.WithError("Failed to enable container runtime", err) } @@ -275,8 +272,8 @@ func setupKubeconfig(h *host.Host, cc *config.ClusterConfig, n *config.Node, clu ClientCertificate: localpath.ClientCert(cc.Name), ClientKey: localpath.ClientKey(cc.Name), CertificateAuthority: localpath.CACert(), - KeepContext: viper.GetBool(keepContext), - EmbedCerts: viper.GetBool(embedCerts), + KeepContext: cc.KeepContext, + EmbedCerts: cc.EmbedCerts, } kcs.SetPath(kubeconfig.PathFromEnv()) @@ -303,7 +300,7 @@ func startMachine(cfg *config.ClusterConfig, node *config.Node) (runner command. exit.WithError("Failed to get command runner", err) } - ip := validateNetwork(host, runner) + ip := validateNetwork(host, runner, cfg.KubernetesConfig.ImageRepository) // Bypass proxy for minikube's vm host ip err = proxy.ExcludeIP(ip) @@ -352,7 +349,7 @@ func startHost(api libmachine.API, cc config.ClusterConfig, n config.Node) (*hos } // validateNetwork tries to catch network problems as soon as possible -func validateNetwork(h *host.Host, r command.Runner) string { +func validateNetwork(h *host.Host, r command.Runner, imageRepository string) string { ip, err := h.Driver.GetIP() if err != nil { exit.WithError("Unable to get VM IP address", err) @@ -381,7 +378,7 @@ func validateNetwork(h *host.Host, r command.Runner) string { } // Non-blocking - go tryRegistry(r, h.Driver.DriverName()) + go tryRegistry(r, h.Driver.DriverName(), imageRepository) return ip } @@ -423,7 +420,7 @@ func trySSH(h *host.Host, ip string) { } // tryRegistry tries to connect to the image repository -func tryRegistry(r command.Runner, driverName string) { +func tryRegistry(r command.Runner, driverName string, imageRepository string) { // 2 second timeout. For best results, call tryRegistry in a non-blocking manner. opts := []string{"-sS", "-m", "2"} @@ -432,15 +429,14 @@ func tryRegistry(r command.Runner, driverName string) { opts = append([]string{"-x", proxy}, opts...) } - repo := viper.GetString(imageRepository) - if repo == "" { - repo = images.DefaultKubernetesRepo + if imageRepository == "" { + imageRepository = images.DefaultKubernetesRepo } - opts = append(opts, fmt.Sprintf("https://%s/", repo)) + opts = append(opts, fmt.Sprintf("https://%s/", imageRepository)) if rr, err := r.RunCmd(exec.Command("curl", opts...)); err != nil { glog.Warningf("%s failed: %v", rr.Args, err) - out.WarningT("This {{.type}} is having trouble accessing https://{{.repository}}", out.V{"repository": repo, "type": driver.MachineType(driverName)}) + out.WarningT("This {{.type}} is having trouble accessing https://{{.repository}}", out.V{"repository": imageRepository, "type": driver.MachineType(driverName)}) out.ErrT(out.Tip, "To pull new external images, you may need to configure a proxy: https://minikube.sigs.k8s.io/docs/reference/networking/proxy/") } } diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 7ddbe72fb59b..9dd63bbc2931 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -38,7 +38,9 @@ import ( "github.com/google/go-cmp/cmp" + "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/localpath" + "k8s.io/minikube/pkg/util/retry" "github.com/elazarl/goproxy" "github.com/hashicorp/go-retryablehttp" @@ -46,12 +48,14 @@ import ( "github.com/phayes/freeport" "github.com/pkg/errors" "golang.org/x/build/kubernetes/api" - "k8s.io/minikube/pkg/util/retry" ) // validateFunc are for subtests that share a single setup type validateFunc func(context.Context, *testing.T, string) +// used in validateStartWithProxy and validateSoftStart +var apiPortTest = 8441 + // TestFunctional are functionality tests which can safely share a profile in parallel func TestFunctional(t *testing.T) { @@ -80,6 +84,7 @@ func TestFunctional(t *testing.T) { }{ {"CopySyncFile", setupFileSync}, // Set file for the file sync test case {"StartWithProxy", validateStartWithProxy}, // Set everything else up for success + {"SoftStart", validateSoftStart}, // do a soft start. ensure config didnt change. {"KubeContext", validateKubeContext}, // Racy: must come immediately after "minikube start" {"KubectlGetPods", validateKubectlGetPods}, // Make sure apiserver is up {"CacheCmd", validateCacheCmd}, // Caches images needed for subsequent tests because of proxy @@ -184,7 +189,8 @@ func validateStartWithProxy(ctx context.Context, t *testing.T, profile string) { } // Use more memory so that we may reliably fit MySQL and nginx - startArgs := append([]string{"start", "-p", profile, "--wait=true"}, StartArgs()...) + // changing api server so later in soft start we verify it didn't change + startArgs := append([]string{"start", "-p", profile, fmt.Sprintf("--apiserver-port=%d", apiPortTest), "--wait=true"}, StartArgs()...) c := exec.CommandContext(ctx, Target(), startArgs...) env := os.Environ() env = append(env, fmt.Sprintf("HTTP_PROXY=%s", srv.Addr)) @@ -206,6 +212,37 @@ func validateStartWithProxy(ctx context.Context, t *testing.T, profile string) { } } +// validateSoftStart validates that after minikube already started, a "minikube start" should not change the configs. +func validateSoftStart(ctx context.Context, t *testing.T, profile string) { + start := time.Now() + // the test before this had been start with --apiserver-port=8441 + beforeCfg, err := config.LoadProfile(profile) + if err != nil { + t.Errorf("error reading cluster config before soft start: %v", err) + } + if beforeCfg.Config.KubernetesConfig.NodePort != apiPortTest { + t.Errorf("expected cluster config node port before soft start to be %d but got %d", apiPortTest, beforeCfg.Config.KubernetesConfig.NodePort) + } + + softStartArgs := []string{"start", "-p", profile} + c := exec.CommandContext(ctx, Target(), softStartArgs...) + rr, err := Run(t, c) + if err != nil { + t.Errorf("failed to soft start minikube. args %q: %v", rr.Command(), err) + } + t.Logf("soft start took %s for %q cluster.", time.Since(start), profile) + + afterCfg, err := config.LoadProfile(profile) + if err != nil { + t.Errorf("error reading cluster config after soft start: %v", err) + } + + if afterCfg.Config.KubernetesConfig.NodePort != apiPortTest { + t.Errorf("expected node port in the config not change after soft start. exepceted node port to be %d but got %d.", apiPortTest, afterCfg.Config.KubernetesConfig.NodePort) + } + +} + // validateKubeContext asserts that kubectl is properly configured (race-condition prone!) func validateKubeContext(ctx context.Context, t *testing.T, profile string) { rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "config", "current-context"))