feat: zfs support

cmd/ctl/helper/machine.go

@@ -18,6 +18,7 @@ func GetMachineInfo() {
 	getWorkDir()
 	getHost()
 	getCpu()
+	getFs()
 	getDisk()
 	getMem()
 	getRepoManager()
@@ -71,6 +72,15 @@ func getDisk() {
 	constants.DiskFree = diskFree
 }
 
+func getFs() {
+	fsType, zfsPrefixName, err := util.GetFs()
+	if err != nil {
+		panic(err)
+	}
+	constants.FsType = fsType
+	constants.DefaultZfsPrefixName = zfsPrefixName
+}
+
 func getMem() {
 	memTotal, memFree, err := util.GetMem()
 	if err != nil {

pkg/bootstrap/precheck/hooks.go

@@ -35,6 +35,7 @@ func (h *PrintMachineInfoHook) Try() error {
 		constants.HostName, constants.CpuPhysicalCount, utils.FormatBytes(int64(constants.MemTotal)),
 		utils.FormatBytes(int64(constants.DiskTotal)), constants.LocalIp)
 	fmt.Printf("SYSTEM, os: %s, platform: %s, arch: %s, version: %s\nKERNEL: version: %s\n", constants.OsType, constants.OsPlatform, constants.OsArch, constants.OsVersion, constants.OsKernel)
+	fmt.Printf("FS, type: %s, zfsmount: %s\n", constants.FsType, constants.DefaultZfsPrefixName)
 	fmt.Printf("VIRTUAL, role: %s, system: %s\n", constants.VirtualizationRole, constants.VirtualizationSystem)
 	fmt.Printf("CGROUP, cpu-enabled: %d, memory-enabled: %d\n", constants.CgroupCpuEnabled, constants.CgroupMemoryEnabled)
 

pkg/bootstrap/precheck/tasks.go

@@ -171,6 +171,7 @@ func (t *GetSysInfoTask) Execute(runtime connector.Runtime) error {
 	logger.Infof("cpu info, model: %s, logical count: %d, physical count: %d",
 		constants.CpuModel, constants.CpuLogicalCount, constants.CpuPhysicalCount)
 	logger.Infof("disk info, total: %s, free: %s", utils.FormatBytes(int64(constants.DiskTotal)), utils.FormatBytes(int64(constants.DiskFree)))
+	logger.Info("fs info, fs: %s, zfsmount: %s", constants.FsType, constants.DefaultZfsPrefixName)
 	logger.Infof("mem info, total: %s, free: %s", utils.FormatBytes(int64(constants.MemTotal)), utils.FormatBytes(int64(constants.MemFree)))
 	logger.Infof("cgroup info, cpu: %d, mem: %d", constants.CgroupCpuEnabled, constants.CgroupMemoryEnabled)
 

pkg/constants/constants.go

@@ -26,6 +26,8 @@ var (
 	MemFree              uint64
 	DiskTotal            uint64
 	DiskFree             uint64
+	FsType               string
+	DefaultZfsPrefixName string
 
 	CgroupCpuEnabled    int
 	CgroupMemoryEnabled int

pkg/container/containerd.go

@@ -25,8 +25,10 @@ import (
 	"time"
 
 	"bytetrade.io/web3os/installer/pkg/common"
+	"bytetrade.io/web3os/installer/pkg/constants"
 	"bytetrade.io/web3os/installer/pkg/container/templates"
 	"bytetrade.io/web3os/installer/pkg/core/action"
+	cc "bytetrade.io/web3os/installer/pkg/core/common"
 	"bytetrade.io/web3os/installer/pkg/core/connector"
 	"bytetrade.io/web3os/installer/pkg/core/logger"
 	"bytetrade.io/web3os/installer/pkg/core/prepare"
@@ -39,6 +41,65 @@ import (
 	"github.com/pkg/errors"
 )
 
+type CreateZfsMount struct {
+	common.KubeAction
+}
+
+func (t *CreateZfsMount) Execute(runtime connector.Runtime) error {
+	if constants.FsType != "zfs" {
+		return nil
+	}
+	var cmd = fmt.Sprintf("zfs create -o mountpoint=%s %s/containerd", cc.ZfsSnapshotter, constants.DefaultZfsPrefixName)
+	if _, err := runtime.GetRunner().SudoCmd(cmd, false, true); err != nil {
+		if strings.Contains(err.Error(), "already exists") {
+			logger.Debugf("zfs %s/containerd already exists", constants.DefaultZfsPrefixName)
+			return nil
+		}
+		logger.Errorf("create zfs mount error %v", err)
+	}
+	return nil
+}
+
+type ZfsReset struct {
+	common.KubeAction
+}
+
+func (t *ZfsReset) Execute(runtime connector.Runtime) error {
+	if _, err := util.GetCommand("zfs"); err != nil {
+		return err
+	}
+
+	res, _ := runtime.GetRunner().SudoCmdExt("zfs list -t all", false, false)
+	if res != "" {
+		scanner := bufio.NewScanner(strings.NewReader(res))
+		for scanner.Scan() {
+			line := scanner.Text()
+			fields := strings.Fields(line)
+			if len(fields) < 5 {
+				continue
+			}
+
+			var name = fields[0]
+
+			if !strings.Contains(name, fmt.Sprintf("%s/containerd", constants.DefaultZfsPrefixName)) {
+				continue
+			}
+			var mp = fields[4]
+			if !strings.Contains(mp, "legacy") {
+				continue
+			}
+
+			if _, err := runtime.GetRunner().SudoCmdExt(fmt.Sprintf("zfs destroy %s -frR", name), false, false); err == nil {
+				fmt.Printf("delete zfs device %s\n", name)
+			}
+		}
+	}
+
+	runtime.GetRunner().SudoCmdExt(fmt.Sprintf("zfs destroy %s/containerd -frR", constants.DefaultZfsPrefixName), false, false)
+
+	return nil
+}
+
 type SyncContainerd struct {
 	common.KubeAction
 }

pkg/container/module.go

@@ -19,13 +19,16 @@ package container
 import (
 	"path/filepath"
 	"strings"
+	"time"
 
+	"bytetrade.io/web3os/installer/pkg/constants"
 	"bytetrade.io/web3os/installer/pkg/kubernetes"
 	"bytetrade.io/web3os/installer/pkg/registry"
 
 	"bytetrade.io/web3os/installer/pkg/common"
 	"bytetrade.io/web3os/installer/pkg/container/templates"
 	"bytetrade.io/web3os/installer/pkg/core/action"
+	cc "bytetrade.io/web3os/installer/pkg/core/common"
 	"bytetrade.io/web3os/installer/pkg/core/logger"
 	"bytetrade.io/web3os/installer/pkg/core/prepare"
 	"bytetrade.io/web3os/installer/pkg/core/task"
@@ -60,6 +63,33 @@ func (i *InstallContainerModule) Init() {
 }
 
 func InstallContainerd(m *InstallContainerModule) []task.Interface {
+	fsReset := &task.RemoteTask{
+		Name:  "DeleteZfsMount",
+		Hosts: m.Runtime.GetHostsByRole(common.K8s),
+		Prepare: &prepare.PrepareCollection{
+			&kubernetes.NodeInCluster{Not: true, NoneCluster: m.NoneCluster},
+			&ContainerdExist{Not: true},
+			&ZfsResetPrepare{},
+		},
+		Action:   new(ZfsReset),
+		Parallel: false,
+		Retry:    5,
+		Delay:    5 * time.Second,
+	}
+
+	createZfsMount := &task.RemoteTask{
+		Name:  "CreateZfsMount",
+		Hosts: m.Runtime.GetHostsByRole(common.K8s),
+		Prepare: &prepare.PrepareCollection{
+			&kubernetes.NodeInCluster{Not: true, NoneCluster: m.NoneCluster},
+			&ContainerdExist{Not: true},
+			&ZfsResetPrepare{},
+		},
+		Action:   new(CreateZfsMount),
+		Parallel: false,
+		Retry:    1,
+	}
+
 	syncContainerd := &task.RemoteTask{
 		Name:  "SyncContainerd",
 		Desc:  "Sync containerd binaries",
@@ -120,6 +150,8 @@ func InstallContainerd(m *InstallContainerModule) []task.Interface {
 				"SandBoxImage":       images.GetImage(m.Runtime, m.KubeConf, "pause").ImageName(),
 				"Auths":              registry.DockerRegistryAuthEntries(m.KubeConf.Cluster.Registry.Auths),
 				"DataRoot":           templates.DataRoot(m.KubeConf),
+				"FsType":             constants.FsType,
+				"ZfsRootPath":        cc.ZfsSnapshotter,
 			},
 		},
 		Parallel: true,
@@ -157,6 +189,8 @@ func InstallContainerd(m *InstallContainerModule) []task.Interface {
 	}
 
 	return []task.Interface{
+		fsReset,
+		createZfsMount,
 		syncContainerd,
 		syncCrictlBinaries,
 		generateContainerdService,
@@ -212,15 +246,13 @@ func UninstallDocker(m *UninstallContainerModule) []task.Interface {
 }
 
 func UninstallContainerd(m *UninstallContainerModule) []task.Interface {
+
 	disableContainerd := &task.RemoteTask{
-		Name:  "UninstallContainerd",
-		Desc:  "Uninstall containerd",
-		Hosts: m.Runtime.GetHostsByRole(common.K8s),
-		// Prepare: &prepare.PrepareCollection{
-		// 	&ContainerdExist{Not: false},
-		// },
+		Name:     "UninstallContainerd",
+		Desc:     "Uninstall containerd",
+		Hosts:    m.Runtime.GetHostsByRole(common.K8s),
 		Action:   new(DisableContainerd),
-		Parallel: true,
+		Parallel: false,
 	}
 
 	return []task.Interface{
@@ -305,3 +337,32 @@ func MigrateACri(p *CriMigrateModule) []task.Interface {
 
 	return p.Tasks
 }
+
+type DeleteZfsMountModule struct {
+	common.KubeModule
+	Skip bool
+}
+
+func (i *DeleteZfsMountModule) IsSkip() bool {
+	return i.Skip
+}
+
+func (m *DeleteZfsMountModule) Init() {
+	m.Name = "DeleteZfsMount"
+
+	zfsReset := &task.RemoteTask{
+		Name:  "DeleteZfsMount",
+		Hosts: m.Runtime.GetHostsByRole(common.K8s),
+		Prepare: &prepare.PrepareCollection{
+			new(ZfsResetPrepare),
+		},
+		Action:   new(ZfsReset),
+		Parallel: false,
+		Retry:    5,
+		Delay:    5 * time.Second,
+	}
+
+	m.Tasks = []task.Interface{
+		zfsReset,
+	}
+}

pkg/container/prepares.go

@@ -20,6 +20,7 @@ import (
 	"strings"
 
 	"bytetrade.io/web3os/installer/pkg/common"
+	"bytetrade.io/web3os/installer/pkg/constants"
 	"bytetrade.io/web3os/installer/pkg/core/connector"
 )
 
@@ -90,3 +91,14 @@ func (p *PrivateRegistryAuth) PreCheck(runtime connector.Runtime) (bool, error)
 	}
 	return true, nil
 }
+
+type ZfsResetPrepare struct {
+	common.KubePrepare
+}
+
+func (p *ZfsResetPrepare) PreCheck(runtime connector.Runtime) (bool, error) {
+	if constants.FsType == "zfs" {
+		return true, nil
+	}
+	return false, nil
+}

pkg/container/templates/containerd_config.go

@@ -113,7 +113,7 @@ state = "/run/containerd"
       discard_unpacked_layers = false
       ignore_rdt_not_enabled_errors = false
       no_pivot = false
-      snapshotter = "overlayfs"
+      snapshotter = "{{ .FsType }}"
 
       [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
         base_runtime_spec = ""
@@ -257,7 +257,7 @@ state = "/run/containerd"
     upperdir_label = false
 
   [plugins."io.containerd.snapshotter.v1.zfs"]
-    root_path = ""
+    root_path = "{{ .ZfsRootPath }}"
 
   [plugins."io.containerd.tracing.processor.v1.otlp"]
     endpoint = ""

pkg/core/common/common.go

@@ -85,3 +85,7 @@ const (
 const (
 	DownloadUrl = "https://dc3p1870nn3cj.cloudfront.net"
 )
+
+const (
+	ZfsSnapshotter = "/var/lib/containerd/io.containerd.snapshotter.v1.zfs"
+)

pkg/core/util/sysinfo.go

@@ -46,6 +46,46 @@ func GetCpu() (string, int, int, error) {
 	return cpuInfo[0].ModelName, cpuLogicalCount, cpuPhysicalCount, nil
 }
 
+func GetFs() (string, string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	// todo support other fs type
+	var fsType = "overlayfs"
+	var zfsPrefixName = ""
+
+	ps, err := disk.PartitionsWithContext(ctx, true)
+	if err != nil {
+		return "", "", err
+	}
+
+	if ps == nil || len(ps) == 0 {
+		return "", "", fmt.Errorf("partitions state is empty")
+	}
+
+	for _, p := range ps {
+		if p.Mountpoint == "/var/lib" && p.Fstype == "zfs" {
+			fsType = "zfs"
+			zfsPrefixName = p.Device
+			break
+		}
+	}
+
+	return fsType, zfsPrefixName, nil
+}
+
+func GetPs() ([]disk.PartitionStat, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	ps, err := disk.PartitionsWithContext(ctx, true)
+	if err != nil {
+		return nil, err
+	}
+
+	return ps, nil
+}
+
 func GetDisk() (uint64, uint64, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
 	defer cancel()

pkg/images/load.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"bytetrade.io/web3os/installer/pkg/common"
+	"bytetrade.io/web3os/installer/pkg/constants"
 	"bytetrade.io/web3os/installer/pkg/core/cache"
 	cc "bytetrade.io/web3os/installer/pkg/core/common"
 	"bytetrade.io/web3os/installer/pkg/core/connector"
@@ -80,7 +81,6 @@ type LoadImages struct {
 }
 
 func (t *LoadImages) Execute(runtime connector.Runtime) (reserr error) {
-
 	var minikubepath = getMinikubePath(t.PipelineCache)
 	var minikubeprofile = t.KubeConf.Arg.MinikubeProfile
 	var containerManager = t.KubeConf.Cluster.Kubernetes.ContainerManager
@@ -115,6 +115,10 @@ func (t *LoadImages) Execute(runtime connector.Runtime) (reserr error) {
 
 	var mf = filterMinikubeImages(runtime.GetRunner(), host.GetOs(), minikubepath, manifests, minikubeprofile)
 	for _, imageRepoTag := range mf {
+		if imageRepoTag == "" {
+			continue
+		}
+
 		reserr = nil
 		if inspectImage(runtime.GetRunner(), containerManager, imageRepoTag) == nil {
 			logger.Debugf("%s already exists", imageRepoTag)
@@ -162,6 +166,11 @@ func (t *LoadImages) Execute(runtime connector.Runtime) (reserr error) {
 
 		var imgFileName = filepath.Base(imageFileName)
 		var loadCmd string
+		var loadParm string
+
+		if constants.FsType == "zfs" {
+			loadParm = "--snapshotter=zfs"
+		}
 
 		if runtime.GetRunner().Host.GetOs() == common.Darwin {
 			if HasSuffixI(imgFileName, ".tar.gz", ".tgz") {
@@ -175,9 +184,9 @@ func (t *LoadImages) Execute(runtime connector.Runtime) (reserr error) {
 				loadCmd = "ctr" // not implement
 			case "containerd":
 				if HasSuffixI(imgFileName, ".tar.gz", ".tgz") {
-					loadCmd = fmt.Sprintf("env PATH=$PATH gunzip -c %s | ctr -n k8s.io images import -", imageFileName)
+					loadCmd = fmt.Sprintf("env PATH=$PATH gunzip -c %s | ctr -n k8s.io images import %s -", imageFileName, loadParm)
 				} else {
-					loadCmd = fmt.Sprintf("env PATH=$PATH ctr -n k8s.io images import %s", imageFileName)
+					loadCmd = fmt.Sprintf("env PATH=$PATH ctr -n k8s.io images import %s %s", imageFileName, loadParm)
 				}
 			case "isula":
 				loadCmd = "isula" // not implement