From ac7e9710ba27e396ba72ec8a9e6ddc3445827c16 Mon Sep 17 00:00:00 2001 From: aby913 Date: Mon, 16 Dec 2024 19:28:14 +0800 Subject: [PATCH] feat: support installing Olares in PVE LXC containers --- pkg/bootstrap/os/module.go | 41 ++++++++++++++ pkg/bootstrap/os/prepares.go | 23 ++++++++ pkg/bootstrap/os/tasks.go | 54 +++++++++++++++++++ pkg/bootstrap/os/templates/init_lxc.go | 38 +++++++++++++ pkg/bootstrap/patch/tasks.go | 19 +++---- pkg/common/common.go | 26 ++++----- pkg/core/common/common.go | 1 + pkg/core/connector/system.go | 15 +++++- pkg/kubernetes/tasks.go | 7 +-- .../templates/v1beta2/kubeadm_config.go | 8 ++- pkg/phase/system/linux.go | 1 + 11 files changed, 207 insertions(+), 26 deletions(-) create mode 100644 pkg/bootstrap/os/templates/init_lxc.go diff --git a/pkg/bootstrap/os/module.go b/pkg/bootstrap/os/module.go index b21457a..af87998 100644 --- a/pkg/bootstrap/os/module.go +++ b/pkg/bootstrap/os/module.go @@ -27,6 +27,47 @@ import ( "bytetrade.io/web3os/installer/pkg/core/util" ) +type PvePatchModule struct { + common.KubeModule + Skip bool +} + +func (p *PvePatchModule) IsSkip() bool { + return p.Skip +} + +func (p *PvePatchModule) Init() { + p.Name = "PvePatch" + + removePveCNDomain := &task.LocalTask{ + Name: "RemovePveCNDomain", + Action: new(RemoveCNDomain), + Prepare: new(IsPve), + Retry: 1, + } + + patchLxcInitScript := &task.LocalTask{ + Name: "PatchLxcInitScript", + Action: new(PatchLxcInitScript), + Prepare: new(IsPveLxc), + Retry: 1, + } + + patchLxcEnvVars := &task.LocalTask{ + Name: "PatchLxcEnvVars", + Action: new(PatchLxcEnvVars), + Prepare: new(IsPveLxc), + Retry: 1, + } + + p.Tasks = []task.Interface{ + removePveCNDomain, + patchLxcInitScript, + patchLxcEnvVars, + } + +} + type ConfigSystemModule struct { common.KubeModule } diff --git a/pkg/bootstrap/os/prepares.go b/pkg/bootstrap/os/prepares.go index c0772d0..0afdcb1 100644 --- a/pkg/bootstrap/os/prepares.go +++ b/pkg/bootstrap/os/prepares.go @@ -85,3 +85,26 @@ func (d *DeleteNode) PreCheck(runtime connector.Runtime) (bool, error) { return false, nil } + +type IsPveLxc struct { + common.KubePrepare +} + +func (r *IsPveLxc) PreCheck(runtime connector.Runtime) (bool, error) { + if runtime.GetSystemInfo().IsPveLxc() { + return true, nil + } + return false, nil +} + +type IsPve struct { + common.KubePrepare +} + +func (r *IsPve) PreCheck(runtime connector.Runtime) (bool, error) { + sys := runtime.GetSystemInfo() + if sys.IsPve() { + return true, nil + } + return false, nil +} diff --git a/pkg/bootstrap/os/tasks.go b/pkg/bootstrap/os/tasks.go index bdc18b5..4c0e5d4 100644 --- a/pkg/bootstrap/os/tasks.go +++ b/pkg/bootstrap/os/tasks.go @@ -20,6 +20,7 @@ import ( "fmt" "io/fs" "io/ioutil" + "os" "path" "path/filepath" "strings" @@ -28,13 +29,66 @@ import ( "github.com/pkg/errors" "bytetrade.io/web3os/installer/pkg/bootstrap/os/repository" + "bytetrade.io/web3os/installer/pkg/bootstrap/os/templates" "bytetrade.io/web3os/installer/pkg/common" + cc "bytetrade.io/web3os/installer/pkg/core/common" "bytetrade.io/web3os/installer/pkg/core/connector" "bytetrade.io/web3os/installer/pkg/core/logger" "bytetrade.io/web3os/installer/pkg/core/util" "bytetrade.io/web3os/installer/pkg/utils" ) +// pve-lxc +type PatchLxcEnvVars struct { + common.KubeAction +} + +func (p *PatchLxcEnvVars) Execute(runtime connector.Runtime) error { + var cmd = `sed -n '/export PATH=\"\/usr\/local\/bin:$PATH\"/p' ~/.bashrc` + if res, _ := runtime.GetRunner().Host.CmdExt(cmd, false, false); res == "" { + if _, err := runtime.GetRunner().Host.Cmd("echo 'export PATH=\"/usr/local/bin:$PATH\"' >> ~/.bashrc", false, false); err != nil { + return err + } + + os.Setenv("PATH", "/usr/local/bin:"+os.Getenv("PATH")) + } + return nil +} + +type PatchLxcInitScript struct { + common.KubeAction +} + +func (p *PatchLxcInitScript) Execute(runtime connector.Runtime) error { + filePath := path.Join("/", "etc", templates.InitPveLxcTmpl.Name()) + + lxcPatchScriptStr, err := util.Render(templates.InitPveLxcTmpl, nil) + if err != nil { + return errors.Wrap(errors.WithStack(err), "render lxc patch template failed") + } + + if err := util.WriteFile(filePath, []byte(lxcPatchScriptStr), cc.FileMode0755); err != nil { + return errors.Wrap(errors.WithStack(err), fmt.Sprintf("write lxc patch %s failed", filePath)) + } + + _, _ = runtime.GetRunner().Host.Cmd("/etc/rc.local", false, false) + return nil +} + +type RemoveCNDomain struct { + common.KubeAction +} + +func (r *RemoveCNDomain) Execute(runtime connector.Runtime) error { + if res, _ := runtime.GetRunner().Host.CmdExt("sed -n '/search/p' /etc/resolv.conf", false, false); res != "" { + if _, err := runtime.GetRunner().Host.CmdExt("sed -i '/search/d' /etc/resolv.conf", false, false); err != nil { + return err + } + } + return nil +} + +// general type UpdateNtpDateTask struct { common.KubeAction } diff --git a/pkg/bootstrap/os/templates/init_lxc.go b/pkg/bootstrap/os/templates/init_lxc.go new file mode 100644 index 0000000..39eed9c --- /dev/null +++ b/pkg/bootstrap/os/templates/init_lxc.go @@ -0,0 +1,38 @@ +/* + Copyright 2021 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package templates + +import ( + "text/template" + + "github.com/lithammer/dedent" +) + +var InitPveLxcTmpl = template.Must(template.New("rc.local").Parse( + dedent.Dedent(`#!/bin/sh -e +if [ ! -e /dev/kmsg ]; then + ln -s /dev/console /dev/kmsg +fi + +if grep -q 'search' /etc/resolv.conf; then + sed -i '/search/d' /etc/resolv.conf +fi + +mount --make-rshared / + +exit 0 +`))) diff --git a/pkg/bootstrap/patch/tasks.go b/pkg/bootstrap/patch/tasks.go index 6a7d696..a19c376 100644 --- a/pkg/bootstrap/patch/tasks.go +++ b/pkg/bootstrap/patch/tasks.go @@ -43,6 +43,15 @@ func (t *PatchTask) Execute(runtime connector.Runtime) error { if _, err := util.GetCommand(common.CommandSudo); err != nil { pre_reqs = pre_reqs + " sudo " } + if _, err := util.GetCommand(common.CommandUpdatePciids); err != nil { + pre_reqs = pre_reqs + " pciutils " + } + if _, err := util.GetCommand(common.CommandIptables); err != nil { + pre_reqs = pre_reqs + " iptables " + } + if _, err := util.GetCommand(common.CommandNmcli); err != nil { + pre_reqs = pre_reqs + " network-manager " + } var systemInfo = runtime.GetSystemInfo() var platformFamily = systemInfo.GetOsPlatformFamily() @@ -65,7 +74,7 @@ func (t *PatchTask) Execute(runtime connector.Runtime) error { fallthrough case common.Ubuntu: if systemInfo.IsUbuntu() { - if !systemInfo.IsPve() && !systemInfo.IsRaspbian() { + if !systemInfo.IsPveOrPveLxc() && !systemInfo.IsRaspbian() { if _, err := runtime.GetRunner().Host.SudoCmd("add-apt-repository universe -y", false, true); err != nil { logger.Errorf("add os repo error %v", err) return err @@ -85,20 +94,12 @@ func (t *PatchTask) Execute(runtime connector.Runtime) error { logger.Debug("apt update success") - // if _, err := runtime.GetRunner().Host.SudoCmd("apt --fix-broken install -y", false, true); err != nil { - // logger.Errorf("fix-broken install error %v", err) - // return err - // } - if _, err := runtime.GetRunner().Host.SudoCmd(fmt.Sprintf("%s install -y -qq %s", pkgManager, pre_reqs), false, true); err != nil { logger.Errorf("install deps %s error %v", pre_reqs, err) return err } var cmd = "conntrack socat apache2-utils ntpdate net-tools make gcc bison flex tree unzip" - if runtime.GetSystemInfo().IsUbuntuVersionEqual(connector.Ubuntu24) { - cmd = cmd + " pciutils" - } if _, err := runtime.GetRunner().Host.SudoCmd(fmt.Sprintf("%s %s install -y %s", debianFrontend, pkgManager, cmd), false, true); err != nil { logger.Errorf("install deps %s error %v", cmd, err) return err diff --git a/pkg/common/common.go b/pkg/common/common.go index 2d04777..45e0818 100755 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -184,18 +184,20 @@ const ( ) const ( - CommandIptables = "iptables" - CommandGPG = "gpg" - CommandSudo = "sudo" - CommandSocat = "socat" - CommandConntrack = "conntrack" - CommandNtpdate = "ntpdate" - CommandHwclock = "hwclock" - CommandKubectl = "kubectl" - CommandDocker = "docker" - CommandMinikube = "minikube" - CommandUnzip = "unzip" - CommandVelero = "velero" + CommandIptables = "iptables" + CommandGPG = "gpg" + CommandSudo = "sudo" + CommandSocat = "socat" + CommandConntrack = "conntrack" + CommandNtpdate = "ntpdate" + CommandHwclock = "hwclock" + CommandKubectl = "kubectl" + CommandDocker = "docker" + CommandMinikube = "minikube" + CommandUnzip = "unzip" + CommandVelero = "velero" + CommandUpdatePciids = "update-pciids" + CommandNmcli = "nmcli" CacheCommandKubectlPath = "kubectl_bin_path" CacheCommandMinikubePath = "minikube_bin_path" diff --git a/pkg/core/common/common.go b/pkg/core/common/common.go index 9424421..1fc05fa 100755 --- a/pkg/core/common/common.go +++ b/pkg/core/common/common.go @@ -82,6 +82,7 @@ const ( Raspbian = "raspbian" WSL = "wsl" PVE = "pve" + PVE_LXC = "pve_lxc" Ubuntu = "ubuntu" Debian = "debian" diff --git a/pkg/core/connector/system.go b/pkg/core/connector/system.go index 3915a50..4405ccc 100644 --- a/pkg/core/connector/system.go +++ b/pkg/core/connector/system.go @@ -68,6 +68,8 @@ type Systems interface { IsDarwin() bool IsWsl() bool IsPve() bool + IsPveLxc() bool + IsPveOrPveLxc() bool IsRaspbian() bool IsLinux() bool @@ -222,7 +224,15 @@ func (s *SystemInfo) IsDarwin() bool { } func (s *SystemInfo) IsPve() bool { - return strings.Contains(s.HostInfo.OsKernel, "-pve") + return s.HostInfo.OsPlatform == common.PVE +} + +func (s *SystemInfo) IsPveLxc() bool { + return s.HostInfo.OsPlatform == common.PVE_LXC +} + +func (s *SystemInfo) IsPveOrPveLxc() bool { + return s.IsPve() || s.IsPveLxc() } func (s *SystemInfo) IsWsl() bool { @@ -406,6 +416,9 @@ func formatOsPlatform(osType, osPlatform, osKernel string) string { } if strings.Contains(osKernel, "pve") { + if lxc := os.Getenv("container"); lxc == "lxc" { + return common.PVE_LXC + } return common.PVE } diff --git a/pkg/kubernetes/tasks.go b/pkg/kubernetes/tasks.go index 9794aec..79b6764 100644 --- a/pkg/kubernetes/tasks.go +++ b/pkg/kubernetes/tasks.go @@ -18,8 +18,6 @@ package kubernetes import ( "bufio" - "bytetrade.io/web3os/installer/pkg/storage" - storagetpl "bytetrade.io/web3os/installer/pkg/storage/templates" "context" "encoding/base64" "fmt" @@ -30,6 +28,9 @@ import ( "strings" "time" + "bytetrade.io/web3os/installer/pkg/storage" + storagetpl "bytetrade.io/web3os/installer/pkg/storage/templates" + "bytetrade.io/web3os/installer/pkg/etcd" "bytetrade.io/web3os/installer/pkg/manifest" @@ -371,7 +372,7 @@ func (g *GenerateKubeadmConfig) Execute(runtime connector.Runtime) error { "ControllerManagerArgs": v1beta2.UpdateFeatureGatesConfiguration(ControllerManagerArgs, g.KubeConf), "SchedulerArgs": v1beta2.UpdateFeatureGatesConfiguration(SchedulerArgs, g.KubeConf), "KubeletConfiguration": v1beta2.GetKubeletConfiguration(runtime, g.KubeConf, g.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint, g.WithSecurityEnhancement), - "KubeProxyConfiguration": v1beta2.GetKubeProxyConfiguration(g.KubeConf), + "KubeProxyConfiguration": v1beta2.GetKubeProxyConfiguration(g.KubeConf, runtime.GetSystemInfo().IsPveLxc()), "IsControlPlane": host.IsRole(common.Master), "CgroupDriver": checkCgroupDriver, "BootstrapToken": bootstrapToken, diff --git a/pkg/kubernetes/templates/v1beta2/kubeadm_config.go b/pkg/kubernetes/templates/v1beta2/kubeadm_config.go index e1b8df4..b1c93b5 100644 --- a/pkg/kubernetes/templates/v1beta2/kubeadm_config.go +++ b/pkg/kubernetes/templates/v1beta2/kubeadm_config.go @@ -391,7 +391,7 @@ func GetKubeletCgroupDriver(runtime connector.Runtime, kubeConf *common.KubeConf return kubeletCgroupDriver, nil } -func GetKubeProxyConfiguration(kubeConf *common.KubeConf) map[string]interface{} { +func GetKubeProxyConfiguration(kubeConf *common.KubeConf, isPveLxc bool) map[string]interface{} { defaultKubeProxyConfiguration := map[string]interface{}{ "clusterCIDR": kubeConf.Cluster.Network.KubePodsCIDR, "mode": kubeConf.Cluster.Kubernetes.ProxyMode, @@ -403,6 +403,12 @@ func GetKubeProxyConfiguration(kubeConf *common.KubeConf) map[string]interface{} }, } + if isPveLxc { + defaultKubeProxyConfiguration["conntrack"] = map[string]interface{}{ + "maxPerCore": 0, + } + } + customKubeProxyConfiguration := make(map[string]interface{}) if len(kubeConf.Cluster.Kubernetes.KubeProxyConfiguration.Raw) != 0 { err := yaml.Unmarshal(kubeConf.Cluster.Kubernetes.KubeProxyConfiguration.Raw, &customKubeProxyConfiguration) diff --git a/pkg/phase/system/linux.go b/pkg/phase/system/linux.go index f546fb6..cbed429 100644 --- a/pkg/phase/system/linux.go +++ b/pkg/phase/system/linux.go @@ -27,6 +27,7 @@ type linuxPhaseBuilder struct { func (l *linuxPhaseBuilder) base() phase { m := []module.Module{ + &os.PvePatchModule{Skip: !l.runtime.GetSystemInfo().IsPveOrPveLxc()}, &precheck.PreCheckOsModule{ ManifestModule: manifest.ManifestModule{ Manifest: l.manifestMap,