feat: support installing Olares in PVE LXC containers

pkg/bootstrap/os/module.go

@@ -27,6 +27,47 @@ import (
 	"bytetrade.io/web3os/installer/pkg/core/util"
 )
 
+type PvePatchModule struct {
+	common.KubeModule
+	Skip bool
+}
+
+func (p *PvePatchModule) IsSkip() bool {
+	return p.Skip
+}
+
+func (p *PvePatchModule) Init() {
+	p.Name = "PvePatch"
+
+	removePveCNDomain := &task.LocalTask{
+		Name:    "RemovePveCNDomain",
+		Action:  new(RemoveCNDomain),
+		Prepare: new(IsPve),
+		Retry:   1,
+	}
+
+	patchLxcInitScript := &task.LocalTask{
+		Name:    "PatchLxcInitScript",
+		Action:  new(PatchLxcInitScript),
+		Prepare: new(IsPveLxc),
+		Retry:   1,
+	}
+
+	patchLxcEnvVars := &task.LocalTask{
+		Name:    "PatchLxcEnvVars",
+		Action:  new(PatchLxcEnvVars),
+		Prepare: new(IsPveLxc),
+		Retry:   1,
+	}
+
+	p.Tasks = []task.Interface{
+		removePveCNDomain,
+		patchLxcInitScript,
+		patchLxcEnvVars,
+	}
+
+}
+
 type ConfigSystemModule struct {
 	common.KubeModule
 }

pkg/bootstrap/os/prepares.go

@@ -85,3 +85,26 @@ func (d *DeleteNode) PreCheck(runtime connector.Runtime) (bool, error) {
 
 	return false, nil
 }
+
+type IsPveLxc struct {
+	common.KubePrepare
+}
+
+func (r *IsPveLxc) PreCheck(runtime connector.Runtime) (bool, error) {
+	if runtime.GetSystemInfo().IsPveLxc() {
+		return true, nil
+	}
+	return false, nil
+}
+
+type IsPve struct {
+	common.KubePrepare
+}
+
+func (r *IsPve) PreCheck(runtime connector.Runtime) (bool, error) {
+	sys := runtime.GetSystemInfo()
+	if sys.IsPve() {
+		return true, nil
+	}
+	return false, nil
+}

pkg/bootstrap/os/tasks.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io/fs"
 	"io/ioutil"
+	"os"
 	"path"
 	"path/filepath"
 	"strings"
@@ -28,13 +29,66 @@ import (
 	"github.com/pkg/errors"
 
 	"bytetrade.io/web3os/installer/pkg/bootstrap/os/repository"
+	"bytetrade.io/web3os/installer/pkg/bootstrap/os/templates"
 	"bytetrade.io/web3os/installer/pkg/common"
+	cc "bytetrade.io/web3os/installer/pkg/core/common"
 	"bytetrade.io/web3os/installer/pkg/core/connector"
 	"bytetrade.io/web3os/installer/pkg/core/logger"
 	"bytetrade.io/web3os/installer/pkg/core/util"
 	"bytetrade.io/web3os/installer/pkg/utils"
 )
 
+// pve-lxc
+type PatchLxcEnvVars struct {
+	common.KubeAction
+}
+
+func (p *PatchLxcEnvVars) Execute(runtime connector.Runtime) error {
+	var cmd = `sed -n '/export PATH=\"\/usr\/local\/bin:$PATH\"/p' ~/.bashrc`
+	if res, _ := runtime.GetRunner().Host.CmdExt(cmd, false, false); res == "" {
+		if _, err := runtime.GetRunner().Host.Cmd("echo 'export PATH=\"/usr/local/bin:$PATH\"' >> ~/.bashrc", false, false); err != nil {
+			return err
+		}
+
+		os.Setenv("PATH", "/usr/local/bin:"+os.Getenv("PATH"))
+	}
+	return nil
+}
+
+type PatchLxcInitScript struct {
+	common.KubeAction
+}
+
+func (p *PatchLxcInitScript) Execute(runtime connector.Runtime) error {
+	filePath := path.Join("/", "etc", templates.InitPveLxcTmpl.Name())
+
+	lxcPatchScriptStr, err := util.Render(templates.InitPveLxcTmpl, nil)
+	if err != nil {
+		return errors.Wrap(errors.WithStack(err), "render lxc patch template failed")
+	}
+
+	if err := util.WriteFile(filePath, []byte(lxcPatchScriptStr), cc.FileMode0755); err != nil {
+		return errors.Wrap(errors.WithStack(err), fmt.Sprintf("write lxc patch %s failed", filePath))
+	}
+
+	_, _ = runtime.GetRunner().Host.Cmd("/etc/rc.local", false, false)
+	return nil
+}
+
+type RemoveCNDomain struct {
+	common.KubeAction
+}
+
+func (r *RemoveCNDomain) Execute(runtime connector.Runtime) error {
+	if res, _ := runtime.GetRunner().Host.CmdExt("sed -n '/search/p' /etc/resolv.conf", false, false); res != "" {
+		if _, err := runtime.GetRunner().Host.CmdExt("sed -i '/search/d' /etc/resolv.conf", false, false); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// general
 type UpdateNtpDateTask struct {
 	common.KubeAction
 }

pkg/bootstrap/os/templates/init_lxc.go

@@ -0,0 +1,38 @@
+/*
+ Copyright 2021 The KubeSphere Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package templates
+
+import (
+	"text/template"
+
+	"github.com/lithammer/dedent"
+)
+
+var InitPveLxcTmpl = template.Must(template.New("rc.local").Parse(
+	dedent.Dedent(`#!/bin/sh -e
+if [ ! -e /dev/kmsg ]; then
+  ln -s /dev/console /dev/kmsg
+fi
+
+if grep -q 'search' /etc/resolv.conf; then
+  sed -i '/search/d' /etc/resolv.conf
+fi
+
+mount --make-rshared /
+
+exit 0
+`)))

pkg/bootstrap/patch/tasks.go

@@ -43,6 +43,15 @@ func (t *PatchTask) Execute(runtime connector.Runtime) error {
 	if _, err := util.GetCommand(common.CommandSudo); err != nil {
 		pre_reqs = pre_reqs + " sudo "
 	}
+	if _, err := util.GetCommand(common.CommandUpdatePciids); err != nil {
+		pre_reqs = pre_reqs + " pciutils "
+	}
+	if _, err := util.GetCommand(common.CommandIptables); err != nil {
+		pre_reqs = pre_reqs + " iptables "
+	}
+	if _, err := util.GetCommand(common.CommandNmcli); err != nil {
+		pre_reqs = pre_reqs + " network-manager "
+	}
 
 	var systemInfo = runtime.GetSystemInfo()
 	var platformFamily = systemInfo.GetOsPlatformFamily()
@@ -65,7 +74,7 @@ func (t *PatchTask) Execute(runtime connector.Runtime) error {
 		fallthrough
 	case common.Ubuntu:
 		if systemInfo.IsUbuntu() {
-			if !systemInfo.IsPve() && !systemInfo.IsRaspbian() {
+			if !systemInfo.IsPveOrPveLxc() && !systemInfo.IsRaspbian() {
 				if _, err := runtime.GetRunner().Host.SudoCmd("add-apt-repository universe -y", false, true); err != nil {
 					logger.Errorf("add os repo error %v", err)
 					return err
@@ -85,20 +94,12 @@ func (t *PatchTask) Execute(runtime connector.Runtime) error {
 
 		logger.Debug("apt update success")
 
-		// if _, err := runtime.GetRunner().Host.SudoCmd("apt --fix-broken install -y", false, true); err != nil {
-		// 	logger.Errorf("fix-broken install error %v", err)
-		// 	return err
-		// }
-
 		if _, err := runtime.GetRunner().Host.SudoCmd(fmt.Sprintf("%s install -y -qq %s", pkgManager, pre_reqs), false, true); err != nil {
 			logger.Errorf("install deps %s error %v", pre_reqs, err)
 			return err
 		}
 
 		var cmd = "conntrack socat apache2-utils ntpdate net-tools make gcc bison flex tree unzip"
-		if runtime.GetSystemInfo().IsUbuntuVersionEqual(connector.Ubuntu24) {
-			cmd = cmd + " pciutils"
-		}
 		if _, err := runtime.GetRunner().Host.SudoCmd(fmt.Sprintf("%s %s install -y %s", debianFrontend, pkgManager, cmd), false, true); err != nil {
 			logger.Errorf("install deps %s error %v", cmd, err)
 			return err

pkg/common/common.go

@@ -184,18 +184,20 @@ const (
 )
 
 const (
-	CommandIptables  = "iptables"
-	CommandGPG       = "gpg"
-	CommandSudo      = "sudo"
-	CommandSocat     = "socat"
-	CommandConntrack = "conntrack"
-	CommandNtpdate   = "ntpdate"
-	CommandHwclock   = "hwclock"
-	CommandKubectl   = "kubectl"
-	CommandDocker    = "docker"
-	CommandMinikube  = "minikube"
-	CommandUnzip     = "unzip"
-	CommandVelero    = "velero"
+	CommandIptables     = "iptables"
+	CommandGPG          = "gpg"
+	CommandSudo         = "sudo"
+	CommandSocat        = "socat"
+	CommandConntrack    = "conntrack"
+	CommandNtpdate      = "ntpdate"
+	CommandHwclock      = "hwclock"
+	CommandKubectl      = "kubectl"
+	CommandDocker       = "docker"
+	CommandMinikube     = "minikube"
+	CommandUnzip        = "unzip"
+	CommandVelero       = "velero"
+	CommandUpdatePciids = "update-pciids"
+	CommandNmcli        = "nmcli"
 
 	CacheCommandKubectlPath  = "kubectl_bin_path"
 	CacheCommandMinikubePath = "minikube_bin_path"

pkg/core/common/common.go

@@ -82,6 +82,7 @@ const (
 	Raspbian = "raspbian"
 	WSL      = "wsl"
 	PVE      = "pve"
+	PVE_LXC  = "pve_lxc"
 
 	Ubuntu = "ubuntu"
 	Debian = "debian"

pkg/core/connector/system.go

@@ -68,6 +68,8 @@ type Systems interface {
 	IsDarwin() bool
 	IsWsl() bool
 	IsPve() bool
+	IsPveLxc() bool
+	IsPveOrPveLxc() bool
 	IsRaspbian() bool
 	IsLinux() bool
 
@@ -222,7 +224,15 @@ func (s *SystemInfo) IsDarwin() bool {
 }
 
 func (s *SystemInfo) IsPve() bool {
-	return strings.Contains(s.HostInfo.OsKernel, "-pve")
+	return s.HostInfo.OsPlatform == common.PVE
+}
+
+func (s *SystemInfo) IsPveLxc() bool {
+	return s.HostInfo.OsPlatform == common.PVE_LXC
+}
+
+func (s *SystemInfo) IsPveOrPveLxc() bool {
+	return s.IsPve() || s.IsPveLxc()
 }
 
 func (s *SystemInfo) IsWsl() bool {
@@ -406,6 +416,9 @@ func formatOsPlatform(osType, osPlatform, osKernel string) string {
 	}
 
 	if strings.Contains(osKernel, "pve") {
+		if lxc := os.Getenv("container"); lxc == "lxc" {
+			return common.PVE_LXC
+		}
 		return common.PVE
 	}
 

pkg/kubernetes/tasks.go

@@ -18,8 +18,6 @@ package kubernetes
 
 import (
 	"bufio"
-	"bytetrade.io/web3os/installer/pkg/storage"
-	storagetpl "bytetrade.io/web3os/installer/pkg/storage/templates"
 	"context"
 	"encoding/base64"
 	"fmt"
@@ -30,6 +28,9 @@ import (
 	"strings"
 	"time"
 
+	"bytetrade.io/web3os/installer/pkg/storage"
+	storagetpl "bytetrade.io/web3os/installer/pkg/storage/templates"
+
 	"bytetrade.io/web3os/installer/pkg/etcd"
 	"bytetrade.io/web3os/installer/pkg/manifest"
 
@@ -371,7 +372,7 @@ func (g *GenerateKubeadmConfig) Execute(runtime connector.Runtime) error {
 				"ControllerManagerArgs":  v1beta2.UpdateFeatureGatesConfiguration(ControllerManagerArgs, g.KubeConf),
 				"SchedulerArgs":          v1beta2.UpdateFeatureGatesConfiguration(SchedulerArgs, g.KubeConf),
 				"KubeletConfiguration":   v1beta2.GetKubeletConfiguration(runtime, g.KubeConf, g.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint, g.WithSecurityEnhancement),
-				"KubeProxyConfiguration": v1beta2.GetKubeProxyConfiguration(g.KubeConf),
+				"KubeProxyConfiguration": v1beta2.GetKubeProxyConfiguration(g.KubeConf, runtime.GetSystemInfo().IsPveLxc()),
 				"IsControlPlane":         host.IsRole(common.Master),
 				"CgroupDriver":           checkCgroupDriver,
 				"BootstrapToken":         bootstrapToken,

pkg/kubernetes/templates/v1beta2/kubeadm_config.go

@@ -391,7 +391,7 @@ func GetKubeletCgroupDriver(runtime connector.Runtime, kubeConf *common.KubeConf
 	return kubeletCgroupDriver, nil
 }
 
-func GetKubeProxyConfiguration(kubeConf *common.KubeConf) map[string]interface{} {
+func GetKubeProxyConfiguration(kubeConf *common.KubeConf, isPveLxc bool) map[string]interface{} {
 	defaultKubeProxyConfiguration := map[string]interface{}{
 		"clusterCIDR": kubeConf.Cluster.Network.KubePodsCIDR,
 		"mode":        kubeConf.Cluster.Kubernetes.ProxyMode,
@@ -403,6 +403,12 @@ func GetKubeProxyConfiguration(kubeConf *common.KubeConf) map[string]interface{}
 		},
 	}
 
+	if isPveLxc {
+		defaultKubeProxyConfiguration["conntrack"] = map[string]interface{}{
+			"maxPerCore": 0,
+		}
+	}
+
 	customKubeProxyConfiguration := make(map[string]interface{})
 	if len(kubeConf.Cluster.Kubernetes.KubeProxyConfiguration.Raw) != 0 {
 		err := yaml.Unmarshal(kubeConf.Cluster.Kubernetes.KubeProxyConfiguration.Raw, &customKubeProxyConfiguration)

pkg/phase/system/linux.go

@@ -27,6 +27,7 @@ type linuxPhaseBuilder struct {
 
 func (l *linuxPhaseBuilder) base() phase {
 	m := []module.Module{
+		&os.PvePatchModule{Skip: !l.runtime.GetSystemInfo().IsPveOrPveLxc()},
 		&precheck.PreCheckOsModule{
 			ManifestModule: manifest.ManifestModule{
 				Manifest: l.manifestMap,