From a5aa64334985f6422c99d1db877f9f15594ecf4d Mon Sep 17 00:00:00 2001
From: Tao He <linzhu.ht@alibaba-inc.com>
Date: Thu, 14 Sep 2023 16:14:34 +0800
Subject: [PATCH 01/22] Support getting FID from GID (#1564)

Signed-off-by: Tao He <linzhu.ht@alibaba-inc.com>
---
 modules/graph/fragment/arrow_fragment.vineyard-mod | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/graph/fragment/arrow_fragment.vineyard-mod b/modules/graph/fragment/arrow_fragment.vineyard-mod
index 496ce505..2a1b795c 100644
--- a/modules/graph/fragment/arrow_fragment.vineyard-mod
+++ b/modules/graph/fragment/arrow_fragment.vineyard-mod
@@ -290,6 +290,8 @@ class [[vineyard]] ArrowFragment
     return IsInnerVertex(u) ? fid_ : vid_parser_.GetFid(GetOuterVertexGid(u));
   }
 
+  fid_t GetFragId(const gid_t& u) const { return vid_parser_.GetFid(u); }
+
   size_t GetTotalNodesNum() const { return vm_ptr_->GetTotalNodesNum(); }
   size_t GetTotalVerticesNum() const { return vm_ptr_->GetTotalNodesNum(); }
   size_t GetTotalVerticesNum(label_id_t label) const {

From 5eefb551d20e7f87a4e2df30590571b5022f536a Mon Sep 17 00:00:00 2001
From: Tao He <linzhu.ht@alibaba-inc.com>
Date: Fri, 15 Sep 2023 14:49:13 +0800
Subject: [PATCH 02/22] Fixes the GetFragId API: should be vid_t (#1566)

Fixes #1564

Signed-off-by: Tao He <linzhu.ht@alibaba-inc.com>
---
 modules/graph/fragment/arrow_fragment.vineyard-mod | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/graph/fragment/arrow_fragment.vineyard-mod b/modules/graph/fragment/arrow_fragment.vineyard-mod
index 2a1b795c..1a01a435 100644
--- a/modules/graph/fragment/arrow_fragment.vineyard-mod
+++ b/modules/graph/fragment/arrow_fragment.vineyard-mod
@@ -290,7 +290,7 @@ class [[vineyard]] ArrowFragment
     return IsInnerVertex(u) ? fid_ : vid_parser_.GetFid(GetOuterVertexGid(u));
   }
 
-  fid_t GetFragId(const gid_t& u) const { return vid_parser_.GetFid(u); }
+  fid_t GetFragId(const vid_t& gid) const { return vid_parser_.GetFid(gid); }
 
   size_t GetTotalNodesNum() const { return vm_ptr_->GetTotalNodesNum(); }
   size_t GetTotalVerticesNum() const { return vm_ptr_->GetTotalNodesNum(); }

From 172d18b35c3772a1dacc93278e1c22c9e939aa9f Mon Sep 17 00:00:00 2001
From: zhuyi1159 <57618314+zhuyi1159@users.noreply.github.com>
Date: Mon, 18 Sep 2023 11:07:16 +0800
Subject: [PATCH 03/22] Clean the unused code and prune vineyardctl dependency
 (#1562)

Part of #1268

Signed-off-by: zhuyi1159 <1159751291@qq.com>
---
 ...{webhook_suite_test.go => webhook_test.go} | 61 +++++++++----------
 k8s/cmd/commands/client/ls_metadatas.go       |  9 ---
 .../deploy/deploy_recover_job_test.go         |  2 -
 k8s/cmd/commands/sidecar/inject_test.go       |  2 -
 k8s/cmd/commands/util/format.go               |  4 --
 k8s/cmd/commands/util/usage/md_docs.go        |  5 --
 .../k8s/{suite_test.go => controller_test.go} | 39 ++++++------
 k8s/go.mod                                    |  4 --
 8 files changed, 45 insertions(+), 81 deletions(-)
 rename k8s/apis/k8s/v1alpha1/{webhook_suite_test.go => webhook_test.go} (78%)
 rename k8s/controllers/k8s/{suite_test.go => controller_test.go} (76%)

diff --git a/k8s/apis/k8s/v1alpha1/webhook_suite_test.go b/k8s/apis/k8s/v1alpha1/webhook_test.go
similarity index 78%
rename from k8s/apis/k8s/v1alpha1/webhook_suite_test.go
rename to k8s/apis/k8s/v1alpha1/webhook_test.go
index af8e64ee..cd9bb277 100644
--- a/k8s/apis/k8s/v1alpha1/webhook_suite_test.go
+++ b/k8s/apis/k8s/v1alpha1/webhook_test.go
@@ -16,13 +16,15 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"io"
 	"net"
 	"path/filepath"
 	"testing"
 	"time"
 
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
+	"github.com/avast/retry-go"
+	"github.com/stretchr/testify/assert"
+	"github.com/v6d-io/v6d/k8s/pkg/log"
 
 	admissionv1beta1 "k8s.io/api/admission/v1beta1"
 	//+kubebuilder:scaffold:imports
@@ -47,45 +49,39 @@ var (
 	cancel    context.CancelFunc
 )
 
-func TestAPIs(t *testing.T) {
-	RegisterFailHandler(Fail)
-
-	RunSpecs(t,
-		"Webhook Suite")
-}
-
-var _ = BeforeSuite(func() {
+func Test_webhook(t *testing.T) {
+	var GinkgoWriter io.Writer
 	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
 
 	ctx, cancel = context.WithCancel(context.TODO())
 
-	By("bootstrapping test environment")
+	log.Info("Bootstrapping test environment")
 	testEnv = &envtest.Environment{
 		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "config", "crd", "bases")},
 		ErrorIfCRDPathMissing: false,
 		WebhookInstallOptions: envtest.WebhookInstallOptions{
-			Paths: []string{filepath.Join("..", "..", "..", "config", "webhook")},
+			Paths: []string{filepath.Join("..", "..", "..", "config", "webhook", "manifests.yaml")},
 		},
 	}
 
 	var err error
 	// cfg is defined in this file globally.
 	cfg, err = testEnv.Start()
-	Expect(err).NotTo(HaveOccurred())
-	Expect(cfg).NotTo(BeNil())
+	assert.NoError(t, err)
+	assert.NotNil(t, cfg)
 
 	scheme := runtime.NewScheme()
 	err = AddToScheme(scheme)
-	Expect(err).NotTo(HaveOccurred())
+	assert.NoError(t, err)
 
 	err = admissionv1beta1.AddToScheme(scheme)
-	Expect(err).NotTo(HaveOccurred())
+	assert.NoError(t, err)
 
 	//+kubebuilder:scaffold:scheme
 
 	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme})
-	Expect(err).NotTo(HaveOccurred())
-	Expect(k8sClient).NotTo(BeNil())
+	assert.NoError(t, err)
+	assert.NotNil(t, k8sClient)
 
 	// start webhook server using Manager
 	webhookInstallOptions := &testEnv.WebhookInstallOptions
@@ -97,26 +93,25 @@ var _ = BeforeSuite(func() {
 		LeaderElection:     false,
 		MetricsBindAddress: "0",
 	})
-	Expect(err).NotTo(HaveOccurred())
+	assert.NoError(t, err)
 
 	err = (&Operation{}).SetupWebhookWithManager(mgr)
-	Expect(err).NotTo(HaveOccurred())
+	assert.NoError(t, err)
 
 	err = (&Sidecar{}).SetupWebhookWithManager(mgr)
-	Expect(err).NotTo(HaveOccurred())
+	assert.NoError(t, err)
 
 	err = (&Backup{}).SetupWebhookWithManager(mgr)
-	Expect(err).NotTo(HaveOccurred())
+	assert.NoError(t, err)
 
 	err = (&Recover{}).SetupWebhookWithManager(mgr)
-	Expect(err).NotTo(HaveOccurred())
+	assert.NoError(t, err)
 
 	//+kubebuilder:scaffold:webhook
 
 	go func() {
-		defer GinkgoRecover()
 		err = mgr.Start(ctx)
-		Expect(err).NotTo(HaveOccurred())
+		assert.NoError(t, err)
 	}()
 
 	// wait for the webhook server to get ready
@@ -126,7 +121,7 @@ var _ = BeforeSuite(func() {
 		webhookInstallOptions.LocalServingHost,
 		webhookInstallOptions.LocalServingPort,
 	)
-	Eventually(func() error {
+	err = retry.Do(func() error {
 		conn, err := tls.DialWithDialer(
 			dialer,
 			"tcp",
@@ -138,12 +133,12 @@ var _ = BeforeSuite(func() {
 		}
 		conn.Close()
 		return nil
-	}).Should(Succeed())
-}, 60)
+	})
+	assert.NoError(t, err)
 
-var _ = AfterSuite(func() {
 	cancel()
-	By("tearing down the test environment")
-	err := testEnv.Stop()
-	Expect(err).NotTo(HaveOccurred())
-})
+	time.Sleep(1 * time.Second)
+	log.Info("tearing down the test environment")
+	err = testEnv.Stop()
+	assert.NoError(t, err)
+}
diff --git a/k8s/cmd/commands/client/ls_metadatas.go b/k8s/cmd/commands/client/ls_metadatas.go
index e19b42ad..6c45af16 100644
--- a/k8s/cmd/commands/client/ls_metadatas.go
+++ b/k8s/cmd/commands/client/ls_metadatas.go
@@ -16,8 +16,6 @@ limitations under the License.
 package client
 
 import (
-	"os"
-
 	"github.com/spf13/cobra"
 	"github.com/v6d-io/v6d/k8s/cmd/commands/flags"
 	"github.com/v6d-io/v6d/k8s/cmd/commands/util"
@@ -89,10 +87,3 @@ func init() {
 	flags.ApplyLsOpts(lsMetadatas)
 	flags.ApplyOutputOpts(lsMetadatas)
 }
-
-func DisableStdout() *os.File {
-	stdout := os.Stdout
-	os.Stdout, _ = os.Open(os.DevNull)
-
-	return stdout
-}
diff --git a/k8s/cmd/commands/deploy/deploy_recover_job_test.go b/k8s/cmd/commands/deploy/deploy_recover_job_test.go
index 65adf9e4..c72183dc 100644
--- a/k8s/cmd/commands/deploy/deploy_recover_job_test.go
+++ b/k8s/cmd/commands/deploy/deploy_recover_job_test.go
@@ -17,7 +17,6 @@ package deploy
 
 import (
 	"context"
-	"fmt"
 	"reflect"
 	"testing"
 
@@ -314,7 +313,6 @@ func Test_getRecoverObjectsFromTemplate_third(t *testing.T) {
 			}
 			for i := range got {
 				if !reflect.DeepEqual(*got[i], *(tt.want)[i]) {
-					fmt.Println(*got[i])
 					t.Errorf("getRecoverObjectsFromTemplate() = %+v, want %+v", got, tt.want)
 				}
 			}
diff --git a/k8s/cmd/commands/sidecar/inject_test.go b/k8s/cmd/commands/sidecar/inject_test.go
index 03ea0439..6f3ba094 100644
--- a/k8s/cmd/commands/sidecar/inject_test.go
+++ b/k8s/cmd/commands/sidecar/inject_test.go
@@ -503,8 +503,6 @@ spec:
 				return
 			}
 			if !reflect.DeepEqual(got, tt.want) {
-				fmt.Println(got)
-				fmt.Println(tt.want)
 				t.Errorf("GetWorkloadObj() = %v, want %v", got, tt.want)
 			}
 		})
diff --git a/k8s/cmd/commands/util/format.go b/k8s/cmd/commands/util/format.go
index 17f2a423..ac0ba586 100644
--- a/k8s/cmd/commands/util/format.go
+++ b/k8s/cmd/commands/util/format.go
@@ -29,10 +29,6 @@ func Examples(examples string) string {
 	return formatter{examples}.trim().tab().indent().string
 }
 
-func Docs(yamls string) string {
-	return formatter{yamls}.doc().string
-}
-
 type formatter struct {
 	string
 }
diff --git a/k8s/cmd/commands/util/usage/md_docs.go b/k8s/cmd/commands/util/usage/md_docs.go
index 886c0837..63baa7c4 100644
--- a/k8s/cmd/commands/util/usage/md_docs.go
+++ b/k8s/cmd/commands/util/usage/md_docs.go
@@ -66,11 +66,6 @@ func printOptions(buf *bytes.Buffer, titleLevel string, cmd *cobra.Command, name
 	return nil
 }
 
-// GenMarkdown creates markdown output.
-func GenMarkdown(cmd *cobra.Command, w io.Writer) error {
-	return GenMarkdownCustom(cmd, w, func(s string) string { return s })
-}
-
 // GenMarkdownCustom creates custom markdown output.
 func GenMarkdownCustom(cmd *cobra.Command, w io.Writer, linkHandler func(string) string) error {
 	cmd.InitDefaultHelpCmd()
diff --git a/k8s/controllers/k8s/suite_test.go b/k8s/controllers/k8s/controller_test.go
similarity index 76%
rename from k8s/controllers/k8s/suite_test.go
rename to k8s/controllers/k8s/controller_test.go
index 57f8c18f..5a9423ba 100644
--- a/k8s/controllers/k8s/suite_test.go
+++ b/k8s/controllers/k8s/controller_test.go
@@ -13,11 +13,12 @@ limitations under the License.
 package k8s
 
 import (
+	"io"
 	"path/filepath"
 	"testing"
+	"time"
 
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
+	"github.com/stretchr/testify/assert"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -27,6 +28,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	k8sv1alpha1 "github.com/v6d-io/v6d/k8s/apis/k8s/v1alpha1"
+	"github.com/v6d-io/v6d/k8s/pkg/log"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -37,16 +39,11 @@ var cfg *rest.Config
 var k8sClient client.Client
 var testEnv *envtest.Environment
 
-func TestAPIs(t *testing.T) {
-	RegisterFailHandler(Fail)
-
-	RunSpecs(t, "Controller Suite")
-}
-
-var _ = BeforeSuite(func() {
+func Test_controller(t *testing.T) {
+	var GinkgoWriter io.Writer
 	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
 
-	By("bootstrapping test environment")
+	log.Info("Bootstrapping test environment")
 	testEnv = &envtest.Environment{
 		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "config", "crd", "bases")},
 		ErrorIfCRDPathMissing: true,
@@ -55,22 +52,20 @@ var _ = BeforeSuite(func() {
 	var err error
 	// cfg is defined in this file globally.
 	cfg, err = testEnv.Start()
-	Expect(err).NotTo(HaveOccurred())
-	Expect(cfg).NotTo(BeNil())
+	assert.NoError(t, err)
+	assert.NotNil(t, cfg)
 
 	err = k8sv1alpha1.AddToScheme(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
+	assert.NoError(t, err)
 
 	//+kubebuilder:scaffold:scheme
 
 	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
-	Expect(err).NotTo(HaveOccurred())
-	Expect(k8sClient).NotTo(BeNil())
+	assert.NoError(t, err)
+	assert.NotNil(t, k8sClient)
 
-}, 60)
-
-var _ = AfterSuite(func() {
-	By("tearing down the test environment")
-	err := testEnv.Stop()
-	Expect(err).NotTo(HaveOccurred())
-})
+	time.Sleep(1 * time.Second)
+	log.Info("tearing down the test environment")
+	err = testEnv.Stop()
+	assert.NoError(t, err)
+}
diff --git a/k8s/go.mod b/k8s/go.mod
index 8074bdd0..01935d1d 100644
--- a/k8s/go.mod
+++ b/k8s/go.mod
@@ -10,8 +10,6 @@ require (
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-logr/logr v1.2.3
 	github.com/olekukonko/tablewriter v0.0.4
-	github.com/onsi/ginkgo v1.16.5
-	github.com/onsi/gomega v1.20.1
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/pflag v1.0.5
@@ -85,7 +83,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/selinux v1.10.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -129,7 +126,6 @@ require (
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
-	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiserver v0.25.10 // indirect

From 2850ff7ca15619c0ae2afcebe8f6998150c1167a Mon Sep 17 00:00:00 2001
From: Tao He <linzhu.ht@alibaba-inc.com>
Date: Mon, 18 Sep 2023 11:41:37 +0800
Subject: [PATCH 04/22] Add our 2022 CNCF annual review docs

Signed-off-by: Tao He <linzhu.ht@alibaba-inc.com>
---
 docs/cncf/2022-vineyard-annual.md | 149 ++++++++++++++++++++++++++++++
 1 file changed, 149 insertions(+)
 create mode 100644 docs/cncf/2022-vineyard-annual.md

diff --git a/docs/cncf/2022-vineyard-annual.md b/docs/cncf/2022-vineyard-annual.md
new file mode 100644
index 00000000..909d49b2
--- /dev/null
+++ b/docs/cncf/2022-vineyard-annual.md
@@ -0,0 +1,149 @@
+# Vineyard 2022 Annual Review
+
+## Background
+
+[Vineyard](https://v6d.io) is an in-memory immutable data manager that provides
+**out-of-the-box high-level** abstraction and **zero-copy in-memory** sharing for
+distributed data in big data tasks, such as graph analytics, numerical computing, and
+machine learning.
+
+**Vineyard provides**:
+
+1. Efficient in-memory data management and zero-copy sharing across different systems.
+2. Out-of-the-box high-level data abstraction for distributed objects (e.g., tensors, tables,
+   graphs) and efficient polyglot support (currently including C++, Python, and Java).
+3. Built-in streaming support for data accessing and across system pipelining.
+4. An extensible driver framework and a set of efficient built-in drivers for eliminating
+   the boilerplate part in computation engines, e.g., I/O, serialization, and checkpointing.
+
+**Alignment with CNCF**:
+
+1. Vineyard builds on Kubernetes for deploying and scaling and the objects are observable
+   in Kubernetes as CRDs.
+2. Vineyard makes efficient zero-copy sharing possible for data-intensive workflows on
+   cloud-native infrastructure by a data-aware Kubernetes scheduler plugin.
+3. Vineyard adopts a immutable object design, which aligns with the immutable infrastructure
+   of the cloud-native environment.
+
+**Vineyard was accepted as a CNCF sandbox project on Apr 28th, 2021.**
+
+## DevStats
+
+> Include a link to your project’s devstats page. We will be looking for signs of consistent
+> or increasing contribution activity. Please feel free to add commentary to add colour to
+> the numbers and graphs we will see on devstats.
+
+- Stargazers and Forks
+  - [https://vineyard.devstats.cncf.io/d/3/stars-and-forks-by-repository?orgId=1&from=now-2y&to=now](https://vineyard.devstats.cncf.io/d/3/stars-and-forks-by-repository?orgId=1&from=now-2y&to=now)
+- Commits per week
+  - [https://vineyard.devstats.cncf.io/d/1/activity-repository-groups?orgId=1&var-period=w&var-repogroups=All&from=now-6M&to=now](https://vineyard.devstats.cncf.io/d/1/activity-repository-groups?orgId=1&var-period=w&var-repogroups=All&from=now-6M&to=now)
+- Contributors and Companies
+  - [https://vineyard.devstats.cncf.io/d/7/companies-contributing-in-repository-groups?orgId=1&var-period=d7&var-repogroup_name=All&from=now-1y&to=now](https://vineyard.devstats.cncf.io/d/7/companies-contributing-in-repository-groups?orgId=1&var-period=d7&var-repogroup_name=All&from=now-1y&to=now)
+
+The vineyard community has grown since the project entered the CNCF sandbox.
+
+- Number of contributors: 11 -> 26
+- Github stars: 300+ -> 600+
+- Github forks: 20+ -> 80+
+- Contributing organizations: 1 -> 12
+
+## Maintainers
+
+> How many maintainers do you have, and which organisations are they from? (Feel free
+> to link to an existing MAINTAINERS file if appropriate.)
+
+We currently have 7 maintainers and 2 committers and have [a maintainer list on Github](https://github.com/v6d-io/v6d/blob/main/MAINTAINERS.md).
+
+- **Initial maintainers**
+
+  | Name | GitHub ID | Affiliation | Email |
+  | --- | --- | --- | --- |
+  | Tao He | [sighingnow](https://github.com/sighingnow) | Alibaba | [linzhu.ht@alibaba-inc.com](mailto:linzhu.ht@alibaba-inc.com) |
+  | Xiaojian Luo | [luoxiaojian](https://github.com/luoxiaojian) | Alibaba | [lxj193371@alibaba-inc.com](mailto:lxj193371@alibaba-inc.com) |
+  | Wenyuan Yu | [wenyuanyu](https://github.com/wenyuanyu) | Alibaba | [wenyuan.ywy@alibaba-inc.com](mailto:wenyuan.ywy@alibaba-inc.com) |
+  | Weibin Zeng | [acezen](https://github.com/acezen) | Alibaba | [qiaozi.zwb@alibaba-inc.com](mailto:qiaozi.zwb@alibaba-inc.com) |
+  | Siyuan Zhang | [siyuan0322](https://github.com/siyuan0322) | Alibaba | [siyuanzhang.zsy@alibaba-inc.com](mailto:siyuanzhang.zsy@alibaba-inc.com) |
+  | Diwen Zhu | [andydiwenzhu](https://github.com/andydiwenzhu) | Alibaba | [diwen.zdw@alibaba-inc.com](mailto:diwen.zdw@alibaba-inc.com) |
+
+- **New maintainers in this year**
+
+  | Name | GitHub ID | Affiliation | Email |
+  | --- | --- | --- | --- |
+  | Ke Meng | [septicmk](https://github.com/septicmk) | Alibaba | [mengke.mk@alibaba-inc.com](mailto:mengke.mk@alibaba-inc.com) |
+
+- **New Committers in this year**
+
+  | Name | GitHub ID | Affiliation | Email |
+  | --- | --- | --- | --- |
+  | Lihong Lin | [linlih](https://github.com/linlih) | PKU | [linlh@stu.pku.edu.cn](mailto:linlh@stu.pku.edu.cn) |
+  | Pei Li | [peilii](https://github.com/peilii) | CMU | [peili.dev@gmail.com](mailto:peili.dev@gmail.com) |
+
+## Adoption
+
+> What do you know about adoption, and how has this changed since your last review / since
+> you joined Sandbox? If you can list companies that are end-users of your project, please
+> do so. (Feel free to link to an existing ADOPTERS file if appropriate.)
+
+We know several cases where vineyard has been adopted in both testing and production environments.
+
+- _GraphScope_: production stage
+  - _GraphScope_ is an open-source graph processing platform.
+  - Vineyard is used in graphscope to provide distributed shared in-memory storage for different
+    graph processing engines.
+- _weilaisudu_: transiting from testing to production stage
+  - _weilaisudu_ is the company behind the project [Mars](https://github.com/mars-project/mars),
+    a distributed scientific computing engine that provides numpy and pandas like API.
+  - Vineyard is used as the shared-memory storage for actors that do computation on chunks.
+- _ESRF_: testing stage
+  - _ESRF_ is a joint research facility situated in France, one of the biggest x-ray science
+    facilities in Europe.
+  - VIneyard is used in the BLISS software to serve as the shared storage between sensors and
+    data processing jobs.
+- _PingAn_: testing stage
+  - _PingAn_ is a large-scale fin-tech company in China.
+  - Vineyard is used in a research platform to support efficient dataset sharing and management
+    among data science researchers.
+
+We have also integrated with the apache-airflow project, which is a workflow orchestration engine
+and has been widely adopted. We have published airflow-vineyard-provider on Astronomer Registry
+and received much feedback from end-users, but we haven't tracked the actual adoption yet.
+
+## Project Goals
+
+> How has the project performed against its goals since the last review? (We won't penalize you
+> if your goals changed for good reasons.)
+
+Vineyard has successfully archived the goal of bringing value to big data analytical workflows
+on Kubernetes. We have shown the gain in an internal project which involves both ETL, graph
+computation, and machine learning jobs.
+
+Our goal hasn't changed since becoming CNCF sandbox project and we are still aiming at supporting
+a more efficient big data analytical workflow on the cloud-native infrastructure. Specifically,
+we'll keep moving towards  following goals in the next year:
+
+1. Providing efficient cross-engine data sharing for data-intensive workflows in Kubernetes
+2. Integrating with projects in the cloud-native community for orchestration and scheduling
+   and integrating with more big data computing engines to improve the end-to-end efficiency.
+3. Building a new cloud-native paradigm for big data applications working  together. By
+   integrating Vineyard, Kubernetes can help orchestrating data and workloads together for
+   better alignment and efficiency.
+
+We still need to do more to engage end-users to show the value-added of the vineyard project.
+
+## CNCF membership
+
+> How can the CNCF help you achieve your upcoming goals?
+
+Vineyard has incredibly benefited from CNCF since accepted as a sandbox project. We believe
+the end-users in the CNCF community are critical for Vineyard to become successful. We have
+submitted proposals for the KubeCon and CNCF Conferences in the past year but got rejected.
+We hope we could have more opportunities to introduce our project to border end-users in the
+CNCF community to increase adoption.
+
+## Incubation
+
+> Do you think that your project meets the [criteria for incubation](https://github.com/cncf/toc/blob/master/process/graduation_criteria.adoc#incubating-stage)?
+
+We think our project vineyard still needs further exploration to get border adoption in the
+production environment and we are looking forward to meeting the incubation criteria in near
+future.

From 4a7284c1b5cdd5260e0473682bc6c7ffba3bce7e Mon Sep 17 00:00:00 2001
From: Tao He <linzhu.ht@alibaba-inc.com>
Date: Mon, 18 Sep 2023 17:13:20 +0800
Subject: [PATCH 05/22] Add our 2023 CNCF annual review docs

Signed-off-by: Tao He <linzhu.ht@alibaba-inc.com>
---
 docs/cncf/2023-vineyard-annual.md | 208 ++++++++++++++++++++++++++++++
 1 file changed, 208 insertions(+)
 create mode 100644 docs/cncf/2023-vineyard-annual.md

diff --git a/docs/cncf/2023-vineyard-annual.md b/docs/cncf/2023-vineyard-annual.md
new file mode 100644
index 00000000..d5c00fea
--- /dev/null
+++ b/docs/cncf/2023-vineyard-annual.md
@@ -0,0 +1,208 @@
+# Vineyard 2023 Annual Review
+
+## Background
+
+[Vineyard](https://v6d.io) is an in-memory immutable data manager that provides
+**out-of-the-box high-level** abstraction and **zero-copy in-memory** sharing for
+distributed data in big data tasks, such as graph analytics, numerical computing, and
+machine learning. Vineyard is design to address the inefficiency of data sharing in
+big data analytical workflows on Kubernetes.
+
+**Vineyard provides**:
+
+1. Efficient in-memory data management and zero-copy sharing across different systems.
+2. Out-of-the-box high-level data abstraction for distributed objects (e.g., tensors, tables,
+   graphs, and distributed datasets) and efficient polyglot support (currently including C++,
+   Python, Go, Rust and Java).
+3. Seamless integration with Kubernetes for cluster deployment and management, workloads
+   orchestration, and observability.
+4. Out-of-the-box integration with workflow orchestration engines (including
+   [Apache Airflow](https://github.com/v6d-io/v6d/tree/main/python/vineyard/contrib/airflow),
+   [Flyte](https://github.com/v6d-io/v6d/tree/main/python/vineyard/contrib/kedro)
+   and [Kubeflow Pipelines](https://github.com/v6d-io/v6d/tree/main/k8s/examples)),
+   providing end-users with a unified and intrusive experience to leverage Vineyard in
+   their data-intensive workflows to improving performance.
+
+**Alignment with CNCF**:
+
+1. Vineyard builds on Kubernetes for deploying and scaling, and the objects are observable
+   in Kubernetes as CRDs.
+2. Vineyard makes efficient zero-copy sharing possible for data-intensive workflows on
+   cloud-native infrastructure by a data-aware Kubernetes scheduler plugin.
+3. Vineyard adopts an immutable object design, which aligns with the immutable infrastructure
+   of the cloud-native environment.
+4. Vineyard aligns with the CNCF effort on helping migrate batching system workflows to cloud
+   native environments.
+
+## Development
+
+### DevStats
+
+> Include a link to your project’s devstats page. We will be looking for signs of consistent
+> or increasing contribution activity. Please feel free to add commentary to add colour to
+> the numbers and graphs we will see on devstats.
+
+- Stargazers and Forks
+  - [https://vineyard.devstats.cncf.io/d/3/stars-and-forks-by-repository?orgId=1&from=now-2y&to=now](https://vineyard.devstats.cncf.io/d/3/stars-and-forks-by-repository?orgId=1&from=now-2y&to=now)
+- Commits per week
+  - [https://vineyard.devstats.cncf.io/d/1/activity-repository-groups?orgId=1&var-period=w&var-repogroups=All&from=now-6M&to=now](https://vineyard.devstats.cncf.io/d/1/activity-repository-groups?orgId=1&var-period=w&var-repogroups=All&from=now-6M&to=now)
+- Contributors and Companies
+  - [https://vineyard.devstats.cncf.io/d/7/companies-contributing-in-repository-groups?orgId=1&var-period=d7&var-repogroup_name=All&from=now-1y&to=now](https://vineyard.devstats.cncf.io/d/7/companies-contributing-in-repository-groups?orgId=1&var-period=d7&var-repogroup_name=All&from=now-1y&to=now)
+
+The vineyard community has grown since the project entered the CNCF sandbox.
+
+- Number of contributors: 26 -> 40
+- Github stars: 600+ -> ~750
+- Github forks: 80+ -> 110+
+
+### Highlights of new features
+
+Vineyard published 8 release (one release about per 1.5 month) since the last annual review.
+The major new features and improvements include:
+
+- Language SDKs in Rust and Go, where the Rust SDK was a collaboration with our external
+  end-user, and enabled users seamlessly and efficiently interoperating their data between
+  Python and Rust.
+- Integration with the workflow engine Kedro, and gained attention in the Kedro community.
+- Vineyard supports the [Apache Hive](https://github.com/apache/hive) data processing engine,
+  letting users can easily connect traditional data processing pipelines built with the
+  Hadoop ecosystem with emerging big-data and AI applications (e.g., applications in the
+  [PyData](https://pandas.pydata.org/community/ecosystem.html) community).
+- A initial version of CSI driver, which helped Vineyard aligned with the Kubernetes platform
+  and enables users to leverage Vineyard in their Kubeflow pipelines to optimize the data
+  sharing between steps with only minor changes to their existing source code.
+
+### Academic Research
+
+We have conducted a series of research work around Vineyard and published the paper
+_Vineyard: Optimizing Data Sharing in Data-Intensive Analytics_ in SIGMOD 2023, a top-tier
+conference in the data management community.
+
+- Wenyuan Yu, Tao He, Lei Wang, Ke Meng, Ye Cao, Diwen Zhu, Sanhong Li, Jingren Zhou.
+  Vineyard: Optimizing Data Sharing in Data-Intensive Analytics. ACM SIG Conference on
+  Management of Data (SIGMOD), industry, 2023. [https://dl.acm.org/doi/10.1145/3589780](https://dl.acm.org/doi/10.1145/3589780).
+
+## Maintainers
+
+> How many maintainers do you have, and which organisations are they from? (Feel free
+> to link to an existing MAINTAINERS file if appropriate.)
+
+We currently have 10 maintainers and 2 committers and have [a maintainer list on Github](https://github.com/v6d-io/v6d/blob/main/MAINTAINERS.md).
+
+- **Initial maintainers**
+
+  | Name | GitHub ID | Affiliation | Email |
+  | --- | --- | --- | --- |
+  | Tao He | [sighingnow](https://github.com/sighingnow) | Alibaba | [linzhu.ht@alibaba-inc.com](mailto:linzhu.ht@alibaba-inc.com) |
+  | Xiaojian Luo | [luoxiaojian](https://github.com/luoxiaojian) | Alibaba | [lxj193371@alibaba-inc.com](mailto:lxj193371@alibaba-inc.com) |
+  | Ke Meng | [septicmk](https://github.com/septicmk) | Alibaba | [mengke.mk@alibaba-inc.com](mailto:mengke.mk@alibaba-inc.com) |
+  | Wenyuan Yu | [wenyuanyu](https://github.com/wenyuanyu) | Alibaba | [wenyuan.ywy@alibaba-inc.com](mailto:wenyuan.ywy@alibaba-inc.com) |
+  | Weibin Zeng | [acezen](https://github.com/acezen) | Alibaba | [qiaozi.zwb@alibaba-inc.com](mailto:qiaozi.zwb@alibaba-inc.com) |
+  | Siyuan Zhang | [siyuan0322](https://github.com/siyuan0322) | Alibaba | [siyuanzhang.zsy@alibaba-inc.com](mailto:siyuanzhang.zsy@alibaba-inc.com) |
+  | Diwen Zhu | [andydiwenzhu](https://github.com/andydiwenzhu) | Alibaba | [diwen.zdw@alibaba-inc.com](mailto:diwen.zdw@alibaba-inc.com) |
+
+- **New maintainers in this year**
+
+  | Name | GitHub ID | Affiliation | Email |
+  | --- | --- | --- | --- |
+  | Ye Cao | [dashanji](https://github.com/dashanji) | Alibaba | [caoye.cao@alibaba-inc.com](mailto:caoye.cao@alibaba-inc.com) |
+  | Shumin Yuan | [vegetableysm](https://github.com/vegetableysm) | Alibaba | [yuanshumin.ysm@alibaba-inc.com](mailto:yuanshumin.ysm@alibaba-inc.com) |
+  | Denghao Li | [lidh15](https://github.com/lidh15) | PingAn Tech | [lidhrandom@gmail.com](mailto:lidhrandom@gmail.com) |
+
+- **New Committers in this year**
+
+  | Name | GitHub ID | Affiliation | Email |
+  | --- | --- | --- | --- |
+  | Lihong Lin | [linlih](https://github.com/linlih) | PKU | [linlh@stu.pku.edu.cn](mailto:linlh@stu.pku.edu.cn) |
+  | Pei Li | [peilii](https://github.com/peilii) | CMU | [peili.dev@gmail.com](mailto:peili.dev@gmail.com) |
+
+## Adoption
+
+> What do you know about adoption, and how has this changed since your last review / since
+> you joined Sandbox? If you can list companies that are end-users of your project, please
+> do so. (Feel free to link to an existing ADOPTERS file if appropriate.)
+
+We have tracked the following two major adoption since [our last annual review]()
+
+- _StartDT (Qidianyun)_: transiting towards production stage
+  - _StartDT_ is a startup company in China, providing a cloud-native data platform for
+    big-data analytics and machine learning applications.
+  - Vineyard is currently used in their Python-centric data processing pipelines to
+    share distributed dataframe artifacts between steps, and help build a composable
+    and efficient data processing platform to end-users.
+
+    Vineyard has passed their eager-evaluation, and they are working on building their
+    distributed data processing platform on top of Vineyard.
+
+- _PingAn Tech_: production stage
+  - _PingAn_ is a large-scale fin-tech company in China.
+  - Vineyard is used in their data science platform to support efficient dataset sharing and
+    management among data science researchers.
+
+    The status of Vineyard in their platform has been transited from testing to production
+    stage, and one of their engineers has become a maintainer of the Vineyard project.
+
+Besides these two major companies, since our last annual review, we have also noticed some other
+questions about using Vineyard in machine learning inference scenarios, but we haven't tracked
+the actual adoption yet.
+
+## Project Goals
+
+> How has the project performed against its goals since the last review? (We won't penalize you
+> if your goals changed for good reasons.)
+
+Vineyard has successfully archived the goals about easing the getting started process for
+end-users from three aspects:
+
+1. Out-of-the-box integration with data processing systems, especially Spark and Hive,
+   the most popular data processing engines in the big data community;
+2. Data processing pipeline orchestration: providing non-intrusive interfaces to help users
+   migrate their existing data processing pipelines to Vineyard on Kubernetes and finally
+   benefit from the efficient data sharing;
+3. Seamless inter-operability with other systems in the cloud-native environments: we invest
+   a lot of effort in the Vineyard operator to help use deploy vineyard along with their
+   workloads in a non-intrusive, declarative way and has tested the functionality with
+   _GraphScope_ in end-users production environments.
+
+Besides, Vineyard has successfully attracted new end-users from the big data community to
+adopt Vineyard in their own data processing platform, and the feedback from the Kedro
+community is also positive.
+
+> What are the current goals of the project? For example, are you working on major new features? Or are you concentrating on adoption or documentation?
+
+Our current goals are mainly focused on the attracting more end-user to adopt Vineyard in
+their scenarios from different domains. Specifically, we are keeping moving towards the
+following goals in the next year:
+
+1. Optimizing our current Kubeflow integration and find more opportunities to evaluate
+   and deploy Vineyard in production machine learning applications;
+2. Publish our integration with the big data processing systems to their end-user
+   community and gather feedback for further improvements;
+3. Seeking more opportunities to evaluate Vineyard in the emerging LLM applications,
+   for both data preprocessing, training, and inference serving to see if Vineyard
+   can bring added value to these applications as where the data cost is usually high;
+4. Getting engaged with the Batch System WG in CNCF to seek opportunities about
+   further collaboration with other projects in CNCF.
+
+## CNCF membership
+
+> How can the CNCF help you achieve your upcoming goals?
+
+Vineyard has incredibly benefited from CNCF since accepted as a sandbox project. We believe
+the end-users in the CNCF community are critical for Vineyard to become successful. With
+the help of CNCF service desk, we have successfully built a new website for Vineyard, which
+is more friendly to end-users. We are also working on components like CSI driver and hope
+that could make the inter-operation with other projects in the CNCF community easier.
+
+We will host a _Project Kiosk_ in this KubeCon China and hope to get more feedback from the
+community, and hope to get more feedback from the community. Furthermore, we hope we could have
+more opportunities to introduce our project to border end-users in the CNCF community to
+increase adoption.
+
+## Incubation
+
+> Do you think that your project meets the [criteria for incubation](https://github.com/cncf/toc/blob/master/process/graduation_criteria.adoc#incubating-stage)?
+
+We think our project vineyard still needs further exploration to get border adoption in the
+end-user's production deployment and gather more feedback, and we are looking forward to
+meeting the incubation criteria in the near future.

From e5ca1c7951a39163aaa22797380f24ee2df137dd Mon Sep 17 00:00:00 2001
From: Ye Cao <caoye.cao@alibaba-inc.com>
Date: Tue, 19 Sep 2023 13:04:45 +0800
Subject: [PATCH 06/22] Design a vineyard csi driver for workloads based on
 kubernetes volumes. (#1533)

- [x] Design the vineyard csi driver framework including the driver logic and deployment manifests.
- [x] Create the soft link of vineyard socket for every PersistentVolumeClaim.
- [x] Map every vineyard object to a PersistentVolumeClaim/PersistentVolume.
- [x] Introduce https://github.com/kubernetes-csi/csi-test to validate the idempotence of vineyard csi driver API.

Fixes parts of #1528

Signed-off-by: Ye Cao <caoye.cao@alibaba-inc.com>
---
 .../templates/csidriver-crd.yaml              | 115 +++++++
 .../templates/deployment.yaml                 |   2 +
 .../templates/manager-rbac.yaml               | 134 ++++++++
 .../templates/recover-crd.yaml                |  12 +
 .../validating-webhook-configuration.yaml     |  20 ++
 docs/tutorials/kubernetes.rst                 |   1 +
 ...g-in-kubeflow-with-vineyard-csi-driver.rst | 311 ++++++++++++++++++
 go/vineyard/go.mod                            |   2 +-
 k8s/.gitignore                                |   1 -
 k8s/Dockerfile                                |   6 +-
 k8s/Makefile                                  |   1 +
 k8s/PROJECT                                   |  12 +
 k8s/apis/k8s/v1alpha1/README.md               |  92 ++++++
 k8s/apis/k8s/v1alpha1/csidriver_types.go      | 149 +++++++++
 k8s/apis/k8s/v1alpha1/csidriver_webhook.go    |  69 ++++
 k8s/apis/k8s/v1alpha1/sidecar_webhook.go      |   3 +-
 k8s/apis/k8s/v1alpha1/webhook_test.go         |   6 +
 .../k8s/v1alpha1/zz_generated.deepcopy.go     | 125 +++++++
 k8s/cmd/README.md                             | 110 ++++++-
 k8s/cmd/commands/csi/csi.go                   |  50 +++
 k8s/cmd/commands/csi/csi_test.go              |  67 ++++
 k8s/cmd/commands/delete/delete.go             |   1 +
 k8s/cmd/commands/delete/delete_csidriver.go   |  61 ++++
 k8s/cmd/commands/deploy/deploy.go             |   7 +-
 k8s/cmd/commands/deploy/deploy_csidriver.go   | 107 ++++++
 k8s/cmd/commands/flags/csi_flags.go           |  38 +++
 k8s/cmd/commands/flags/csidriver_flags.go     |  73 ++++
 k8s/cmd/commands/manager/manager.go           |  13 +-
 k8s/cmd/commands/util/parse.go                |  21 ++
 k8s/cmd/main.go                               |   2 +
 .../crd/bases/k8s.v6d.io_csidrivers.yaml      |  97 ++++++
 k8s/config/crd/kustomization.yaml             |   7 +-
 .../patches/cainjection_in_csidrivers.yaml    |   7 +
 ...vers.yaml => cainjection_in_recovers.yaml} |   0
 .../crd/patches/webhook_in_csidrivers.yaml    |  16 +
 ...recovers.yaml => webhook_in_recovers.yaml} |   0
 k8s/config/manager/manager.yaml               |   1 +
 .../rbac/k8s_csidriver_editor_role.yaml       |  24 ++
 .../rbac/k8s_csidriver_viewer_role.yaml       |  20 ++
 k8s/config/rbac/manager_account.yaml          |   6 -
 k8s/config/rbac/role.yaml                     | 134 ++++++++
 .../samples/k8s_v1alpha1_csidriver.yaml       |  10 +
 k8s/config/webhook/manifests.yaml             |  20 ++
 k8s/controllers/k8s/csidriver_controller.go   | 219 ++++++++++++
 k8s/examples/vineyard-csidriver/Dockerfile    |  12 +
 k8s/examples/vineyard-csidriver/Makefile      |  22 ++
 .../pipeline-with-vineyard.py                 |  82 +++++
 .../pipeline-with-vineyard.yaml               | 264 +++++++++++++++
 k8s/examples/vineyard-csidriver/pipeline.py   |  45 +++
 k8s/examples/vineyard-csidriver/pipeline.yaml |  84 +++++
 .../vineyard-csidriver/prepare-data.yaml      |  54 +++
 .../prepare-data/prepare-data.py              |  68 ++++
 .../preprocess/preprocess.py                  |  85 +++++
 k8s/examples/vineyard-csidriver/rbac.yaml     |  30 ++
 k8s/examples/vineyard-csidriver/test/test.py  |  42 +++
 .../vineyard-csidriver/train/train.py         |  40 +++
 k8s/go.mod                                    |  12 +-
 k8s/go.sum                                    |  15 +-
 k8s/pkg/csidriver/controller.go               | 269 +++++++++++++++
 k8s/pkg/csidriver/driver.go                   | 115 +++++++
 k8s/pkg/csidriver/identity.go                 |  70 ++++
 k8s/pkg/csidriver/node.go                     | 183 +++++++++++
 k8s/pkg/csidriver/state.go                    | 127 +++++++
 k8s/pkg/schedulers/scheduling_strategy.go     |   1 +
 k8s/pkg/templates/csidriver/daemonset.yaml    | 112 +++++++
 k8s/pkg/templates/csidriver/deployment.yaml   | 106 ++++++
 k8s/pkg/templates/csidriver/storageclass.yaml |  10 +
 k8s/pkg/templates/template.go                 |   2 +-
 k8s/test/csidriver/Dockerfile                 |  25 ++
 k8s/test/csidriver/Makefile                   |  32 ++
 k8s/test/e2e/Makefile                         |   3 +-
 71 files changed, 4056 insertions(+), 26 deletions(-)
 create mode 100644 charts/vineyard-operator/templates/csidriver-crd.yaml
 create mode 100644 docs/tutorials/kubernetes/efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst
 create mode 100644 k8s/apis/k8s/v1alpha1/csidriver_types.go
 create mode 100644 k8s/apis/k8s/v1alpha1/csidriver_webhook.go
 create mode 100644 k8s/cmd/commands/csi/csi.go
 create mode 100644 k8s/cmd/commands/csi/csi_test.go
 create mode 100644 k8s/cmd/commands/delete/delete_csidriver.go
 create mode 100644 k8s/cmd/commands/deploy/deploy_csidriver.go
 create mode 100644 k8s/cmd/commands/flags/csi_flags.go
 create mode 100644 k8s/cmd/commands/flags/csidriver_flags.go
 create mode 100644 k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml
 create mode 100644 k8s/config/crd/patches/cainjection_in_csidrivers.yaml
 rename k8s/config/crd/patches/{cainjection_in_k8s_recovers.yaml => cainjection_in_recovers.yaml} (100%)
 create mode 100644 k8s/config/crd/patches/webhook_in_csidrivers.yaml
 rename k8s/config/crd/patches/{webhook_in_k8s_recovers.yaml => webhook_in_recovers.yaml} (100%)
 create mode 100644 k8s/config/rbac/k8s_csidriver_editor_role.yaml
 create mode 100644 k8s/config/rbac/k8s_csidriver_viewer_role.yaml
 create mode 100644 k8s/config/samples/k8s_v1alpha1_csidriver.yaml
 create mode 100644 k8s/controllers/k8s/csidriver_controller.go
 create mode 100644 k8s/examples/vineyard-csidriver/Dockerfile
 create mode 100644 k8s/examples/vineyard-csidriver/Makefile
 create mode 100644 k8s/examples/vineyard-csidriver/pipeline-with-vineyard.py
 create mode 100644 k8s/examples/vineyard-csidriver/pipeline-with-vineyard.yaml
 create mode 100644 k8s/examples/vineyard-csidriver/pipeline.py
 create mode 100644 k8s/examples/vineyard-csidriver/pipeline.yaml
 create mode 100644 k8s/examples/vineyard-csidriver/prepare-data.yaml
 create mode 100644 k8s/examples/vineyard-csidriver/prepare-data/prepare-data.py
 create mode 100644 k8s/examples/vineyard-csidriver/preprocess/preprocess.py
 create mode 100644 k8s/examples/vineyard-csidriver/rbac.yaml
 create mode 100644 k8s/examples/vineyard-csidriver/test/test.py
 create mode 100644 k8s/examples/vineyard-csidriver/train/train.py
 create mode 100644 k8s/pkg/csidriver/controller.go
 create mode 100644 k8s/pkg/csidriver/driver.go
 create mode 100644 k8s/pkg/csidriver/identity.go
 create mode 100644 k8s/pkg/csidriver/node.go
 create mode 100644 k8s/pkg/csidriver/state.go
 create mode 100644 k8s/pkg/templates/csidriver/daemonset.yaml
 create mode 100644 k8s/pkg/templates/csidriver/deployment.yaml
 create mode 100644 k8s/pkg/templates/csidriver/storageclass.yaml
 create mode 100644 k8s/test/csidriver/Dockerfile
 create mode 100644 k8s/test/csidriver/Makefile

diff --git a/charts/vineyard-operator/templates/csidriver-crd.yaml b/charts/vineyard-operator/templates/csidriver-crd.yaml
new file mode 100644
index 00000000..87ecf588
--- /dev/null
+++ b/charts/vineyard-operator/templates/csidriver-crd.yaml
@@ -0,0 +1,115 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: csidrivers.k8s.v6d.io
+  annotations:
+    cert-manager.io/inject-ca-from: '{{ .Release.Namespace }}/{{ include "vineyard-operator.fullname"
+      . }}-serving-cert'
+    controller-gen.kubebuilder.io/version: v0.11.0
+  labels:
+  {{- include "vineyard-operator.labels" . | nindent 4 }}
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          name: webhook-service
+          namespace: '{{ .Release.Namespace }}'
+          path: /convert
+      conversionReviewVersions:
+      - v1
+  group: k8s.v6d.io
+  names:
+    kind: CSIDriver
+    listKind: CSIDriverList
+    plural: csidrivers
+    singular: csidriver
+  scope: Cluster
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        properties:
+          apiVersion:
+            type: string
+          kind:
+            type: string
+          metadata:
+            type: object
+          spec:
+            properties:
+              clusters:
+                items:
+                  properties:
+                    name:
+                      default: ""
+                      type: string
+                    namespace:
+                      default: ""
+                      type: string
+                  type: object
+                type: array
+              enableToleration:
+                default: false
+                type: boolean
+              enableVerboseLog:
+                default: false
+                type: boolean
+              image:
+                default: vineyardcloudnative/vineyard-operator
+                type: string
+              imagePullPolicy:
+                default: IfNotPresent
+                type: string
+              sidecar:
+                default:
+                  attacherImage: registry.k8s.io/sig-storage/csi-attacher:v4.0.0
+                  enableTopology: false
+                  imagePullPolicy: Always
+                  livenessProbeImage: registry.k8s.io/sig-storage/livenessprobe:v2.8.0
+                  nodeRegistrarImage: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.0
+                  provisionerImage: registry.k8s.io/sig-storage/csi-provisioner:v3.3.0
+                properties:
+                  attacherImage:
+                    default: registry.k8s.io/sig-storage/csi-attacher:v4.0.0
+                    type: string
+                  enableTopology:
+                    default: false
+                    type: boolean
+                  imagePullPolicy:
+                    default: Always
+                    type: string
+                  livenessProbeImage:
+                    default: registry.k8s.io/sig-storage/livenessprobe:v2.8.0
+                    type: string
+                  nodeRegistrarImage:
+                    default: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.0
+                    type: string
+                  provisionerImage:
+                    default: registry.k8s.io/sig-storage/csi-provisioner:v3.3.0
+                    type: string
+                type: object
+              storageClassName:
+                default: vineyard-csi
+                type: string
+              volumeBindingMode:
+                default: WaitForFirstConsumer
+                type: string
+            type: object
+          status:
+            properties:
+              state:
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
\ No newline at end of file
diff --git a/charts/vineyard-operator/templates/deployment.yaml b/charts/vineyard-operator/templates/deployment.yaml
index 3fe861cf..fb77ec81 100644
--- a/charts/vineyard-operator/templates/deployment.yaml
+++ b/charts/vineyard-operator/templates/deployment.yaml
@@ -11,6 +11,7 @@ metadata:
   name: {{ include "vineyard-operator.fullname" . }}-controller-manager
   labels:
     control-plane: controller-manager
+    k8s.v6d.io/instance: vineyard-operator
   {{- include "vineyard-operator.labels" . | nindent 4 }}
 spec:
   replicas: {{ .Values.controllerManager.replicas }}
@@ -30,6 +31,7 @@ spec:
       containers:
       - args:
         - manager
+        - --verbose
         command:
         - /vineyardctl
         env:
diff --git a/charts/vineyard-operator/templates/manager-rbac.yaml b/charts/vineyard-operator/templates/manager-rbac.yaml
index 0c65aa49..f52f5206 100644
--- a/charts/vineyard-operator/templates/manager-rbac.yaml
+++ b/charts/vineyard-operator/templates/manager-rbac.yaml
@@ -23,6 +23,14 @@ rules:
   verbs:
   - create
   - patch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - ""
   resources:
@@ -34,6 +42,13 @@ rules:
   - list
   - patch
   - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims/finalizers
+  verbs:
+  - patch
 - apiGroups:
   - ""
   resources:
@@ -45,6 +60,13 @@ rules:
   - list
   - patch
   - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes/finalizers
+  verbs:
+  - patch
 - apiGroups:
   - ""
   resources:
@@ -90,6 +112,16 @@ rules:
   - delete
   - get
   - update
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - create
+  - get
+  - list
+  - update
+  - watch
 - apiGroups:
   - apps
   resources:
@@ -127,6 +159,14 @@ rules:
   - get
   - list
   - update
+- apiGroups:
+  - csi.storage.k8s.io
+  resources:
+  - csinodeinfos
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - k8s.v6d.io
   resources:
@@ -147,6 +187,32 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - k8s.v6d.io
+  resources:
+  - csidrivers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - k8s.v6d.io
+  resources:
+  - csidrivers/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - k8s.v6d.io
+  resources:
+  - csidrivers/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - k8s.v6d.io
   resources:
@@ -285,6 +351,74 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - update
+  - watch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshots
+  verbs:
+  - get
+  - list
+  - update
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - csinodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - storageclasses
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments/status
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
diff --git a/charts/vineyard-operator/templates/recover-crd.yaml b/charts/vineyard-operator/templates/recover-crd.yaml
index ff13fe11..ee55908a 100644
--- a/charts/vineyard-operator/templates/recover-crd.yaml
+++ b/charts/vineyard-operator/templates/recover-crd.yaml
@@ -3,10 +3,22 @@ kind: CustomResourceDefinition
 metadata:
   name: recovers.k8s.v6d.io
   annotations:
+    cert-manager.io/inject-ca-from: '{{ .Release.Namespace }}/{{ include "vineyard-operator.fullname"
+      . }}-serving-cert'
     controller-gen.kubebuilder.io/version: v0.11.0
   labels:
   {{- include "vineyard-operator.labels" . | nindent 4 }}
 spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          name: webhook-service
+          namespace: '{{ .Release.Namespace }}'
+          path: /convert
+      conversionReviewVersions:
+      - v1
   group: k8s.v6d.io
   names:
     kind: Recover
diff --git a/charts/vineyard-operator/templates/validating-webhook-configuration.yaml b/charts/vineyard-operator/templates/validating-webhook-configuration.yaml
index b8d0cbb2..ff4e2687 100644
--- a/charts/vineyard-operator/templates/validating-webhook-configuration.yaml
+++ b/charts/vineyard-operator/templates/validating-webhook-configuration.yaml
@@ -27,6 +27,26 @@ webhooks:
     resources:
     - backups
   sideEffects: None
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: '{{ include "vineyard-operator.fullname" . }}-webhook-service'
+      namespace: '{{ .Release.Namespace }}'
+      path: /validate-k8s-v6d-io-v1alpha1-csidriver
+  failurePolicy: Fail
+  name: vcsidriver.kb.io
+  rules:
+  - apiGroups:
+    - k8s.v6d.io
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - csidrivers
+  sideEffects: None
 - admissionReviewVersions:
   - v1
   clientConfig:
diff --git a/docs/tutorials/kubernetes.rst b/docs/tutorials/kubernetes.rst
index d689e590..a7146e3d 100644
--- a/docs/tutorials/kubernetes.rst
+++ b/docs/tutorials/kubernetes.rst
@@ -9,6 +9,7 @@ Vineyard on Kubernetes
    ./kubernetes/using-vineyard-operator.rst
    ./kubernetes/ml-pipeline-mars-pytorch.rst
    ./kubernetes/data-sharing-with-vineyard-on-kubernetes.rst
+   ./kubernetes/efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst
 
 Vineyard can be seamlessly deployed on Kubernetes, managed by the :ref:`vineyard-operator`,
 to enhance big-data workflows through its data-aware scheduling policy. This policy
diff --git a/docs/tutorials/kubernetes/efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst b/docs/tutorials/kubernetes/efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst
new file mode 100644
index 00000000..b8305455
--- /dev/null
+++ b/docs/tutorials/kubernetes/efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst
@@ -0,0 +1,311 @@
+Efficient data sharing in Kubeflow with Vineyard CSI Driver
+===========================================================
+
+If you are using `Kubeflow Pipeline`_ or `Argo Workflow`_ to manage your machine learning workflow, 
+you may find that the data saving/loading to the volumes is slow.
+To speed up the data saving/loading within these volumes, we design the Vineyard CSI Driver to
+map each vineyard object to a volume, and the data saving/loading is handled by vineyard.
+Next, we will show you how to use the Vineyard CSI Driver to speed up a kubeflow pipeline.
+
+Prerequisites
+-------------
+
+- A kubernetes cluster with version >= 1.25.10. If you don't have one by hand, you can refer to the 
+guide `Initialize Kubernetes Cluster`_ to create one.
+- Install the `Vineyardctl`_ by following the official guide.
+- Install the `Argo Workflow CLI`_ by following the official guide.
+
+Deploy the Vineyard Cluster
+---------------------------
+
+.. code:: bash
+
+    $ vineyardctl deploy vineyard-cluster --create-namespace
+
+This command will create a vineyard cluster in the namespace `vineyard-system`.
+You can check as follows:
+
+.. code:: bash
+
+    $ kubectl get pod -n vineyard-system
+    NAME                                             READY   STATUS    RESTARTS   AGE
+    vineyard-controller-manager-648fc9b7bf-zwnhd     2/2     Running   0          4d3h
+    vineyardd-sample-79c8ffb879-6k8mk                1/1     Running   0          4d3h
+    vineyardd-sample-79c8ffb879-f9kkr                1/1     Running   0          4d3h
+    vineyardd-sample-79c8ffb879-lzgwz                1/1     Running   0          4d3h
+    vineyardd-sample-etcd-0                          1/1     Running   0          4d3h
+
+Deploy the Vineyard CSI Driver
+------------------------------
+
+Before deploying the Vineyard CSI Driver, you are supposed to check the vineyard 
+deployment is ready as follows:
+
+.. code:: bash
+
+    $ kubectl get deployment -n vineyard-system        
+    NAME                             READY   UP-TO-DATE   AVAILABLE   AGE
+    vineyard-controller-manager      1/1     1            1           4d3h
+    vineyardd-sample                 3/3     3            3           4d3h
+
+Then deploy the vineyard csi driver which specifies the vineyard cluster to use:
+
+.. code:: bash
+
+    $ vineyardctl deploy csidriver --clusters vineyard-system/vineyardd-sample
+
+Then check the status of the Vineyard CSI Driver:
+
+.. code:: bash
+
+    $ kubectl get pod -n vineyard-system
+    NAME                                             READY   STATUS    RESTARTS   AGE
+    vineyard-controller-manager-648fc9b7bf-zwnhd     2/2     Running   0          4d3h
+    vineyard-csi-sample-csi-driver-fb7cb5b5d-nlrxs   4/4     Running   0          4m23s
+    vineyard-csi-sample-csi-nodes-69j77              3/3     Running   0          4m23s
+    vineyard-csi-sample-csi-nodes-k85hb              3/3     Running   0          4m23s
+    vineyard-csi-sample-csi-nodes-zhfz4              3/3     Running   0          4m23s
+    vineyardd-sample-79c8ffb879-6k8mk                1/1     Running   0          4d3h
+    vineyardd-sample-79c8ffb879-f9kkr                1/1     Running   0          4d3h
+    vineyardd-sample-79c8ffb879-lzgwz                1/1     Running   0          4d3h
+    vineyardd-sample-etcd-0                          1/1     Running   0          4d3h
+
+Deploy Argo Workflows
+---------------------
+
+Install the argo server on Kubernetes:
+
+.. code:: bash
+
+    $ kubectl create namespace argo
+    $ kubectl apply -n argo -f https://github.com/argoproj/argo-workflows/releases/download/v3.4.8/install.yaml
+
+Then check the status of the argo server:
+
+.. code:: bash
+
+    $ kubectl get pod -n argo           
+    NAME                                  READY   STATUS    RESTARTS   AGE
+    argo-server-7698c96655-ft6sj          1/1     Running   0          4d1h
+    workflow-controller-b888f4458-sfrjd   1/1     Running   0          4d1h
+
+Running a Kubeflow Pipeline example
+-----------------------------------
+
+The example is under the directory `k8s/examples/vineyard-csidriver`, and `pipeline.py` under this
+directory is the original pipeline definition. To use the Vineyard CSI Driver, we need to do two 
+modifications:
+
+1. Change APIs like `pd.read_pickle/write_pickle` to `vineyard.csi.write/read` in the source code.
+
+2. Add the `vineyard object` VolumeOp to the pipeline's dependencies. The path in the API changed 
+in the first step will be mapped to a volume. Notice, the volume used in any task needs to be 
+explicitly mounted to the corresponding path in the source code, and the storageclass_name 
+format of each VolumeOp is `{vineyard-deployment-namespace}.{vineyard-deployment-name}.csi`.
+
+You may get some insights from the modified pipeline `pipeline-with-vineyard.py`. Then, we need to
+compile the pipeline to an argo-workflow yaml. To be compatible with benchmark test, we update the
+generated `pipeline.yaml` and `pipeline-with-vineyard.yaml`.
+
+Now, we can build the docker images for the pipeline:
+
+.. code:: bash
+
+    $ cd k8s/examples/vineyard-csidriver
+    $ make docker-build
+
+Check the images built successfully:
+
+.. code:: bash
+
+    $ docker images
+    train-data               latest    5628953ffe08   14 seconds ago   1.47GB
+    test-data                latest    94c8c75b960a   14 seconds ago   1.47GB
+    prepare-data             latest    5aab1b120261   15 seconds ago   1.47GB
+    preprocess-data          latest    5246d09e6f5e   15 seconds ago   1.47GB
+
+Then push the image to a docker registry that your kubernetes cluster can access, as
+we use the kind cluster in this example, we can load the image to the clusters:
+
+.. code:: bash
+
+    $ make load-images
+
+To simulate the data loading/saving of the actual pipeline, we use the nfs volume
+to store the data. The nfs volume is mounted to the `/mnt/data` directory of the 
+kind cluster. Then apply the data volume as follows:
+
+.. tip::
+
+    If you already have nfs volume that can be accessed by the kubernetes cluster,
+    you can update the prepare-data.yaml to use your nfs volume.
+
+.. code:: bash
+
+    $ kubectl apply -f prepare-data.yaml
+
+Deploy the rbac for the pipeline:
+
+.. code:: bash
+
+    $ kubectl apply -f rbac.yaml
+
+Submit the kubeflow example without vineyard to the argo server:
+
+.. code:: bash
+
+    $ for data_multiplier in 3000 4000 5000; do \
+        argo submit --watch pipeline.yaml -p data_multiplier=${data_multiplier}; \
+    done
+
+Clear the previous resources:
+
+.. code:: bash
+
+    $ argo delete --all
+
+Submit the kubeflow example with vineyard to the argo server:
+
+.. code:: bash
+
+    $ for data_multiplier in 3000 4000 5000; do \
+        argo submit --watch pipeline-with-vineyard.yaml -p data_multiplier=${data_multiplier}; \
+    done
+
+Result Analysis
+---------------
+
++------------+------------------+---------------+
+| data scale | without vineyard | with vineyard |
++============+==================+===============+
+| 8500 Mi    | 21s              | 5.4s          |
+| 12000 Mi   | 26s              | 7s            |
+| 15000 Mi   | 32.2s            | 9.4s          |
++------------+------------------+---------------+
+
+The data scale are 8500 Mi, 12000 Mi and 15000 Mi, which correspond to 
+the 3000, 4000 and 5000 in the previous data_multiplier respectively, 
+and the time of argo workflow execution of the pipeline is as follows:
+
+Argo workflow duration
+======================
+
++------------+------------------+---------------+
+| data scale | without vineyard | with vineyard |
++============+==================+===============+
+| 8500 Mi    | 186s             | 169s          |
+| 12000 Mi   | 250s             | 203s          |
+| 15000 Mi   | 332s             | 286s          |
++------------+------------------+---------------+
+
+
+Actually, the cost time of argo workflow is affected by lots of factors, 
+such as the network, the cpu and memory of the cluster, the data volume, etc.
+So the time of argo workflow execution of the pipeline is not stable. 
+But we can still find that the time of argo workflow execution of the pipeline
+with vineyard is shorter than that without vineyard.
+
+Also, we record the whole execution time via logs. The result is as follows:
+
+Actual execution time
+=====================
+
++------------+------------------+---------------+
+| data scale | without vineyard | with vineyard |
++============+==================+===============+
+| 8500 Mi    | 139.3s           | 92.3s         |
+| 12000 Mi   | 204.3s           | 131.1s        |
+| 15000 Mi   | 289.3s           | 209.7s        |
++------------+------------------+---------------+
+
+
+According to the above results, we can find that the time of actual 
+execution of the pipeline with vineyard is shorter than that without vineyard.
+To be specific, we record the write/read time of the following steps:
+
+Writing time
+============
+
++------------+------------------+---------------+
+| data scale | without vineyard | with vineyard |
++============+==================+===============+
+| 8500 Mi    | 21s              | 5.4s          |
+| 12000 Mi   | 26s              | 7s            |
+| 15000 Mi   | 32.2s            | 9.4s          |
++------------+------------------+---------------+
+
+
+From the above results, we can find that the writing time the pipeline 
+with vineyard is nearly 4 times shorter than that without vineyard. 
+The reason is that the data is stored in the vineyard cluster, 
+so it's actually a memory copy operation, which is faster than the 
+write operation of the nfs volume.
+
+
+Reading time
+============
+
+We delete the time of init data loading, and the results are as follows:
+
++------------+------------------+---------------+
+| data scale | without vineyard | with vineyard |
++============+==================+===============+
+| 8500 Mi    | 36.7s            | 0.02s         |
+| 12000 Mi   | 45.7s            | 0.02s         |
+| 15000 Mi   | 128.6s           | 0.04s         |
++------------+------------------+---------------+
+
+Based on the above results, we can find that the read time of vineyard is
+nearly a constant, which is not affected by the data scale.
+The reason is that the data is stored in the shared memory of vineyard cluster, 
+so it's actually a pointer copy operation.
+
+As a result, we can find that with vineyard, the argo workflow 
+duration of the pipeline is reduced by 10%~20% and the actual 
+execution time of the pipeline is reduced by about 30%.
+
+
+Clean up
+--------
+
+Delete the rbac for the kubeflow example:
+
+.. code:: bash
+
+    $ kubectl delete -f rbac.yaml
+
+Delete all argo workflow
+
+.. code:: bash
+
+    $ argo delete --all
+
+Delete the argo server:
+
+.. code:: bash
+
+    $ kubectl delete ns argo
+
+Delete the csi driver:
+
+.. code:: bash
+
+    $ vineyardctl delete csidriver
+
+Delete the vineyard cluster:
+
+.. code:: bash
+
+    $ vineyardctl delete vineyard-cluster
+
+Delete the data volume:
+
+.. code:: bash
+
+    $ kubectl delete -f prepare-data.yaml
+
+.. _Kubeflow Pipeline: https://github.com/kubeflow/kubeflow
+.. _Argo Workflow: https://github.com/argoproj/argo-workflows
+.. _Initialize Kubernetes Cluster: https://v6d.io/tutorials/kubernetes/using-vineyard-operator.html#step-0-optional-initialize-kubernetes-cluster
+.. _Vineyardctl: https://v6d.io/notes/developers/build-from-source.html#install-vineyardctl
+.. _Argo Workflow CLI: https://github.com/argoproj/argo-workflows/releases/
\ No newline at end of file
diff --git a/go/vineyard/go.mod b/go/vineyard/go.mod
index 32b4d330..725a4cff 100644
--- a/go/vineyard/go.mod
+++ b/go/vineyard/go.mod
@@ -1,6 +1,6 @@
 module github.com/v6d-io/v6d/go/vineyard
 
-go 1.18
+go 1.19
 
 require (
 	github.com/apache/arrow/go/v11 v11.0.0
diff --git a/k8s/.gitignore b/k8s/.gitignore
index b84dbcc3..73eb9826 100644
--- a/k8s/.gitignore
+++ b/k8s/.gitignore
@@ -26,4 +26,3 @@ vineyardctl
 
 # vendor, used for code-generate only
 /vendor/
-
diff --git a/k8s/Dockerfile b/k8s/Dockerfile
index ac27a89f..a67504e5 100644
--- a/k8s/Dockerfile
+++ b/k8s/Dockerfile
@@ -21,14 +21,12 @@ RUN go mod download
 RUN go build -a -o vineyardctl cmd/main.go && \
     strip vineyardctl || true
 
+
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
-FROM gcr.io/distroless/static:nonroot
+FROM gcr.io/distroless/base:debug
 WORKDIR /
 
 COPY k8s/config/scheduler/config.yaml /etc/kubernetes/scheduler.yaml
 
 COPY --from=builder /workspace/k8s/vineyardctl /vineyardctl
-USER nonroot:nonroot
-
-ENTRYPOINT ["/vineyardctl", "manager"]
diff --git a/k8s/Makefile b/k8s/Makefile
index 86cb596d..ab232e5e 100644
--- a/k8s/Makefile
+++ b/k8s/Makefile
@@ -29,6 +29,7 @@ BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL)
 # Image URL to use all building/pushing image targets
 REGISTRY	:= vineyardcloudnative
 IMG 		?= $(REGISTRY)/vineyard-operator:$(VERSION)
+VINEYARD_CSI_IMAGE ?= $(REGISTRY)/vineyard-csi-driver:$(VERSION)
 
 temp=$(shell mktemp -d)
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
diff --git a/k8s/PROJECT b/k8s/PROJECT
index f1a47f28..174effef 100644
--- a/k8s/PROJECT
+++ b/k8s/PROJECT
@@ -96,4 +96,16 @@ resources:
     defaulting: true
     validation: true
     webhookVersion: v1
+- api:
+    crdVersion: v1
+  controller: true
+  domain: v6d.io
+  group: k8s
+  kind: CSIDriver
+  path: github.com/v6d-io/v6d/k8s/apis/k8s/v1alpha1
+  version: v1alpha1
+  webhooks:
+    defaulting: true
+    validation: true
+    webhookVersion: v1
 version: "3"
diff --git a/k8s/apis/k8s/v1alpha1/README.md b/k8s/apis/k8s/v1alpha1/README.md
index 0fc51655..7e8d6410 100644
--- a/k8s/apis/k8s/v1alpha1/README.md
+++ b/k8s/apis/k8s/v1alpha1/README.md
@@ -11,6 +11,8 @@ Package v1alpha1 contains API Schema definitions for the k8s v1alpha1 API group
 ### Resource Types
 - [Backup](#backup)
 - [BackupList](#backuplist)
+- [CSIDriver](#csidriver)
+- [CSIDriverList](#csidriverlist)
 - [GlobalObject](#globalobject)
 - [GlobalObjectList](#globalobjectlist)
 - [LocalObject](#localobject)
@@ -80,6 +82,81 @@ _Appears in:_
 
 
 
+#### CSIDriver
+
+
+
+CSIDriver is the Schema for the csidrivers API
+
+_Appears in:_
+- [CSIDriverList](#csidriverlist)
+
+| Field | Description |
+| --- | --- |
+| `apiVersion` _string_ | `k8s.v6d.io/v1alpha1`
+| `kind` _string_ | `CSIDriver`
+| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. |
+| `spec` _[CSIDriverSpec](#csidriverspec)_ |  |
+
+
+#### CSIDriverList
+
+
+
+CSIDriverList contains a list of CSIDriver
+
+
+
+| Field | Description |
+| --- | --- |
+| `apiVersion` _string_ | `k8s.v6d.io/v1alpha1`
+| `kind` _string_ | `CSIDriverList`
+| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. |
+| `items` _[CSIDriver](#csidriver) array_ |  |
+
+
+#### CSIDriverSpec
+
+
+
+CSIDriverSpec defines the desired state of CSIDriver
+
+_Appears in:_
+- [CSIDriver](#csidriver)
+
+| Field | Description |
+| --- | --- |
+| `image` _string_ | Image is the name of the csi driver image |
+| `imagePullPolicy` _string_ | ImagePullPolicy is the image pull policy of the csi driver |
+| `storageClassName` _string_ | StorageClassName is the name of the storage class |
+| `volumeBindingMode` _string_ | VolumeBindingMode is the volume binding mode of the storage class |
+| `sidecar` _[CSISidecar](#csisidecar)_ | Sidecar is the configuration for the CSI sidecar container nolint: lll |
+| `clusters` _[VineyardClusters](#vineyardclusters) array_ | Clusters are the list of vineyard clusters |
+| `enableToleration` _boolean_ | EnableToleration is the flag to enable toleration for the csi driver |
+| `enableVerboseLog` _boolean_ | EnableVerboseLog is the flag to enable verbose log for the csi driver |
+
+
+
+
+#### CSISidecar
+
+
+
+CSISidecar holds the configuration for the CSI sidecar container
+
+_Appears in:_
+- [CSIDriverSpec](#csidriverspec)
+
+| Field | Description |
+| --- | --- |
+| `provisionerImage` _string_ | ProvisionerImage is the image of the provisioner sidecar |
+| `attacherImage` _string_ | AttacherImage is the image of the attacher sidecar |
+| `nodeRegistrarImage` _string_ | NodeRegistrarImage is the image of the node registrar sidecar |
+| `livenessProbeImage` _string_ | LivenessProbeImage is the image of the liveness probe sidecar |
+| `imagePullPolicy` _string_ | ImagePullPolicy is the image pull policy of all sidecar containers |
+| `enableTopology` _boolean_ | EnableTopology is the flag to enable topology for the csi driver |
+
+
 #### GlobalObject
 
 
@@ -419,6 +496,21 @@ _Appears in:_
 | `persistentVolumeClaimSpec` _[PersistentVolumeClaimSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#persistentvolumeclaimspec-v1-core)_ | the PersistentVolumeClaimSpec of the spill file |
 
 
+#### VineyardClusters
+
+
+
+VineyardClusters contains the list of vineyard clusters
+
+_Appears in:_
+- [CSIDriverSpec](#csidriverspec)
+
+| Field | Description |
+| --- | --- |
+| `namespace` _string_ | Namespace is the namespace of the vineyard cluster |
+| `name` _string_ | Name is the name of the vineyard deployment |
+
+
 #### VineyardConfig
 
 
diff --git a/k8s/apis/k8s/v1alpha1/csidriver_types.go b/k8s/apis/k8s/v1alpha1/csidriver_types.go
new file mode 100644
index 00000000..ab10d80f
--- /dev/null
+++ b/k8s/apis/k8s/v1alpha1/csidriver_types.go
@@ -0,0 +1,149 @@
+/** Copyright 2020-2023 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// CSISidecar holds the configuration for the CSI sidecar container
+type CSISidecar struct {
+	// ProvisionerImage is the image of the provisioner sidecar
+	// +kubebuilder:validation:Required
+	// +kubebuilder:default:="registry.k8s.io/sig-storage/csi-provisioner:v3.3.0"
+	ProvisionerImage string `json:"provisionerImage,omitempty"`
+
+	// AttacherImage is the image of the attacher sidecar
+	// +kubebuilder:validation:Required
+	// +kubebuilder:default:="registry.k8s.io/sig-storage/csi-attacher:v4.0.0"
+	AttacherImage string `json:"attacherImage,omitempty"`
+
+	// NodeRegistrarImage is the image of the node registrar sidecar
+	// +kubebuilder:validation:Required
+	// +kubebuilder:default:="registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.0"
+	NodeRegistrarImage string `json:"nodeRegistrarImage,omitempty"`
+
+	// LivenessProbeImage is the image of the liveness probe sidecar
+	// +kubebuilder:validation:Required
+	// +kubebuilder:default:="registry.k8s.io/sig-storage/livenessprobe:v2.8.0"
+	LivenessProbeImage string `json:"livenessProbeImage,omitempty"`
+
+	// ImagePullPolicy is the image pull policy of all sidecar containers
+	// +kubebuilder:validation:Required
+	// +kubebuilder:default:="Always"
+	ImagePullPolicy string `json:"imagePullPolicy,omitempty"`
+
+	// EnableTopology is the flag to enable topology for the csi driver
+	// +kubebuilder:validation:Required
+	// +kubebuilder:default:=false
+	EnableTopology bool `json:"enableTopology,omitempty"`
+}
+
+// VineyardClusters contains the list of vineyard clusters
+type VineyardClusters struct {
+	// Namespace is the namespace of the vineyard cluster
+	// +kubebuilder:validation:Required
+	// +kubebuilder:default:=""
+	Namespace string `json:"namespace,omitempty"`
+
+	// Name is the name of the vineyard deployment
+	// +kubebuilder:validation:Required
+	// +kubebuilder:default:=""
+	Name string `json:"name,omitempty"`
+}
+
+// CSIDriverSpec defines the desired state of CSIDriver
+type CSIDriverSpec struct {
+	// Image is the name of the csi driver image
+	// +kubebuilder:validation:Required
+	// +kubebuilder:default:="vineyardcloudnative/vineyard-operator"
+	Image string `json:"image,omitempty"`
+
+	// ImagePullPolicy is the image pull policy of the csi driver
+	// +kubebuilder:validation:Required
+	// +kubebuilder:default:="IfNotPresent"
+	ImagePullPolicy string `json:"imagePullPolicy,omitempty"`
+
+	// StorageClassName is the name of the storage class
+	// +kubebuilder:validation:Required
+	// +kubebuilder:default:="vineyard-csi"
+	StorageClassName string `json:"storageClassName,omitempty"`
+
+	// VolumeBindingMode is the volume binding mode of the storage class
+	// +kubebuilder:validation:Required
+	// +kubebuilder:default:="WaitForFirstConsumer"
+	VolumeBindingMode string `json:"volumeBindingMode,omitempty"`
+
+	// Sidecar is the configuration for the CSI sidecar container
+	// +kubebuilder:validation:Required
+	//nolint: lll
+	// +kubebuilder:default:={provisionerImage: "registry.k8s.io/sig-storage/csi-provisioner:v3.3.0", attacherImage: "registry.k8s.io/sig-storage/csi-attacher:v4.0.0", nodeRegistrarImage: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.0", livenessProbeImage: "registry.k8s.io/sig-storage/livenessprobe:v2.8.0", imagePullPolicy: "Always", enableTopology: false}
+	Sidecar CSISidecar `json:"sidecar,omitempty"`
+
+	// Clusters are the list of vineyard clusters
+	// +kubebuilder:validation:Required
+	// +kubebuilder:default:={}
+	Clusters []VineyardClusters `json:"clusters,omitempty"`
+
+	// EnableToleration is the flag to enable toleration for the csi driver
+	// +kubebuilder:validation:Required
+	// +kubebuilder:default:=false
+	EnableToleration bool `json:"enableToleration,omitempty"`
+
+	// EnableVerboseLog is the flag to enable verbose log for the csi driver
+	// +kubebuilder:validation:Required
+	// +kubebuilder:default:=false
+	EnableVerboseLog bool `json:"enableVerboseLog,omitempty"`
+}
+
+// CSIDriverStatus defines the observed state of CSIDriver
+type CSIDriverStatus struct {
+	// State is the state of the csi driver
+	State string `json:"state,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+//+kubebuilder:resource:scope=Cluster
+
+// CSIDriver is the Schema for the csidrivers API
+type CSIDriver struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   CSIDriverSpec   `json:"spec,omitempty"`
+	Status CSIDriverStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// CSIDriverList contains a list of CSIDriver
+type CSIDriverList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []CSIDriver `json:"items"`
+}
+
+const (
+	// CSIDriverRunning is the running state of the csi driver
+	CSIDriverRunning = "running"
+	// CSIDriverPending is the pending state of the csi driver
+	CSIDriverPending = "pending"
+)
+
+func init() {
+	SchemeBuilder.Register(&CSIDriver{}, &CSIDriverList{})
+}
diff --git a/k8s/apis/k8s/v1alpha1/csidriver_webhook.go b/k8s/apis/k8s/v1alpha1/csidriver_webhook.go
new file mode 100644
index 00000000..ada36263
--- /dev/null
+++ b/k8s/apis/k8s/v1alpha1/csidriver_webhook.go
@@ -0,0 +1,69 @@
+/** Copyright 2020-2023 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+
+	"github.com/v6d-io/v6d/k8s/pkg/log"
+)
+
+// log is for logging in this package.
+var csidriverlog = log.WithName("csidriver-resource")
+
+func (r *CSIDriver) SetupWebhookWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewWebhookManagedBy(mgr).
+		For(r).
+		Complete()
+}
+
+// nolint: lll
+// +kubebuilder:webhook:path=/mutate-k8s-v6d-io-v1alpha1-csidriver,mutating=true,failurePolicy=fail,sideEffects=None,groups=k8s.v6d.io,resources=csidrivers,verbs=create;update,versions=v1alpha1,name=mcsidriver.kb.io,admissionReviewVersions=v1
+var _ webhook.Defaulter = &CSIDriver{}
+
+// Default implements webhook.Defaulter so a webhook will be registered for the type
+func (r *CSIDriver) Default() {
+	csidriverlog.Info("default", "name", r.Name)
+
+}
+
+//nolint: lll
+//+kubebuilder:webhook:path=/validate-k8s-v6d-io-v1alpha1-csidriver,mutating=false,failurePolicy=fail,sideEffects=None,groups=k8s.v6d.io,resources=csidrivers,verbs=create;update,versions=v1alpha1,name=vcsidriver.kb.io,admissionReviewVersions=v1
+
+var _ webhook.Validator = &CSIDriver{}
+
+// ValidateCreate implements webhook.Validator so a webhook will be registered for the type
+func (r *CSIDriver) ValidateCreate() error {
+	csidriverlog.Info("validate create", "name", r.Name)
+
+	return nil
+}
+
+// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type
+func (r *CSIDriver) ValidateUpdate(old runtime.Object) error {
+	csidriverlog.Info("validate update", "name", r.Name)
+
+	return nil
+}
+
+// ValidateDelete implements webhook.Validator so a webhook will be registered for the type
+func (r *CSIDriver) ValidateDelete() error {
+	csidriverlog.Info("validate delete", "name", r.Name)
+
+	return nil
+}
diff --git a/k8s/apis/k8s/v1alpha1/sidecar_webhook.go b/k8s/apis/k8s/v1alpha1/sidecar_webhook.go
index 154212be..4f6a7716 100644
--- a/k8s/apis/k8s/v1alpha1/sidecar_webhook.go
+++ b/k8s/apis/k8s/v1alpha1/sidecar_webhook.go
@@ -16,10 +16,11 @@ limitations under the License.
 package v1alpha1
 
 import (
-	"github.com/v6d-io/v6d/k8s/pkg/log"
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
+
+	"github.com/v6d-io/v6d/k8s/pkg/log"
 )
 
 // log is for logging in this package.
diff --git a/k8s/apis/k8s/v1alpha1/webhook_test.go b/k8s/apis/k8s/v1alpha1/webhook_test.go
index cd9bb277..2d741af8 100644
--- a/k8s/apis/k8s/v1alpha1/webhook_test.go
+++ b/k8s/apis/k8s/v1alpha1/webhook_test.go
@@ -1,8 +1,11 @@
 /** Copyright 2020-2023 Alibaba Group Holding Limited.
+
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
+
     http://www.apache.org/licenses/LICENSE-2.0
+
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -107,6 +110,9 @@ func Test_webhook(t *testing.T) {
 	err = (&Recover{}).SetupWebhookWithManager(mgr)
 	assert.NoError(t, err)
 
+	err = (&CSIDriver{}).SetupWebhookWithManager(mgr)
+	assert.NoError(t, err)
+
 	//+kubebuilder:scaffold:webhook
 
 	go func() {
diff --git a/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go b/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go
index 21b5c749..7a482e8b 100644
--- a/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go
+++ b/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go
@@ -122,6 +122,116 @@ func (in *BackupStatus) DeepCopy() *BackupStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CSIDriver) DeepCopyInto(out *CSIDriver) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	out.Status = in.Status
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIDriver.
+func (in *CSIDriver) DeepCopy() *CSIDriver {
+	if in == nil {
+		return nil
+	}
+	out := new(CSIDriver)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *CSIDriver) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CSIDriverList) DeepCopyInto(out *CSIDriverList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]CSIDriver, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIDriverList.
+func (in *CSIDriverList) DeepCopy() *CSIDriverList {
+	if in == nil {
+		return nil
+	}
+	out := new(CSIDriverList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *CSIDriverList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CSIDriverSpec) DeepCopyInto(out *CSIDriverSpec) {
+	*out = *in
+	out.Sidecar = in.Sidecar
+	if in.Clusters != nil {
+		in, out := &in.Clusters, &out.Clusters
+		*out = make([]VineyardClusters, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIDriverSpec.
+func (in *CSIDriverSpec) DeepCopy() *CSIDriverSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(CSIDriverSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CSIDriverStatus) DeepCopyInto(out *CSIDriverStatus) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIDriverStatus.
+func (in *CSIDriverStatus) DeepCopy() *CSIDriverStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(CSIDriverStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CSISidecar) DeepCopyInto(out *CSISidecar) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSISidecar.
+func (in *CSISidecar) DeepCopy() *CSISidecar {
+	if in == nil {
+		return nil
+	}
+	out := new(CSISidecar)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *GlobalObject) DeepCopyInto(out *GlobalObject) {
 	*out = *in
@@ -647,6 +757,21 @@ func (in *SpillConfig) DeepCopy() *SpillConfig {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VineyardClusters) DeepCopyInto(out *VineyardClusters) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VineyardClusters.
+func (in *VineyardClusters) DeepCopy() *VineyardClusters {
+	if in == nil {
+		return nil
+	}
+	out := new(VineyardClusters)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *VineyardConfig) DeepCopyInto(out *VineyardConfig) {
 	*out = *in
diff --git a/k8s/cmd/README.md b/k8s/cmd/README.md
index 7df513f5..4fb33561 100644
--- a/k8s/cmd/README.md
+++ b/k8s/cmd/README.md
@@ -13,6 +13,7 @@ drivers.
 **SEE ALSO**
 
 * [vineyardctl create](#vineyardctl-create)	 - Create a vineyard jobs on kubernetes
+* [vineyardctl csi](#vineyardctl-csi)	 - Start the vineyard csi driver
 * [vineyardctl delete](#vineyardctl-delete)	 - Delete the vineyard components from kubernetes
 * [vineyardctl deploy](#vineyardctl-deploy)	 - Deploy the vineyard components on kubernetes
 * [vineyardctl get](#vineyardctl-get)	 - Get vineyard object, metadata, blob or cluster-info
@@ -270,6 +271,34 @@ vineyardctl create recover [flags]
       --recover-name string   the name of recover job (default "vineyard-recover")
 ```
 
+## `vineyardctl csi`
+
+Start the vineyard csi driver
+
+```
+vineyardctl csi [flags]
+```
+
+**SEE ALSO**
+
+* [vineyardctl](#vineyardctl)	 - vineyardctl is the command-line tool for interact with the Vineyard Operator.
+
+### Examples
+
+```shell
+  # start the csi with the specific endpoint and node id
+  vineyardctl csi --endpoint=unix:///csi/csi.sock --nodeid=csinode1
+```
+
+### Options
+
+```
+  -f, --endpoint string          the endpoint of vineyard csi driver
+  -h, --help                     help for csi
+      --nodeid string            the node id of vineyard csi driver
+      --state-file-path string   the path of state file (default "/csi/state")
+```
+
 ## `vineyardctl delete`
 
 Delete the vineyard components from kubernetes
@@ -279,6 +308,7 @@ Delete the vineyard components from kubernetes
 * [vineyardctl](#vineyardctl)	 - vineyardctl is the command-line tool for interact with the Vineyard Operator.
 * [vineyardctl delete backup](#vineyardctl-delete-backup)	 - Delete the backup job on kubernetes
 * [vineyardctl delete cert-manager](#vineyardctl-delete-cert-manager)	 - Delete the cert-manager on kubernetes
+* [vineyardctl delete csidriver](#vineyardctl-delete-csidriver)	 - Delete the vineyard csi driver on kubernetes
 * [vineyardctl delete operation](#vineyardctl-delete-operation)	 - Delete the operation from kubernetes
 * [vineyardctl delete operator](#vineyardctl-delete-operator)	 - Delete the vineyard operator from kubernetes
 * [vineyardctl delete recover](#vineyardctl-delete-recover)	 - Delete the recover job from kubernetes
@@ -368,6 +398,32 @@ vineyardctl delete cert-manager [flags]
   -h, --help   help for cert-manager
 ```
 
+## `vineyardctl delete csidriver`
+
+Delete the vineyard csi driver on kubernetes
+
+```
+vineyardctl delete csidriver [flags]
+```
+
+**SEE ALSO**
+
+* [vineyardctl delete](#vineyardctl-delete)	 - Delete the vineyard components from kubernetes
+
+### Examples
+
+```shell
+  # delete the csi driver named "csidriver-test"
+  vineyardctl delete csidriver --name csidriver-test
+```
+
+### Options
+
+```
+  -h, --help          help for csidriver
+      --name string   The name of the csi driver cr. (default "csidriver-sample")
+```
+
 ## `vineyardctl delete operation`
 
 Delete the operation from kubernetes
@@ -541,6 +597,7 @@ Deploy the vineyard components on kubernetes
 * [vineyardctl](#vineyardctl)	 - vineyardctl is the command-line tool for interact with the Vineyard Operator.
 * [vineyardctl deploy backup-job](#vineyardctl-deploy-backup-job)	 - Deploy a backup job of vineyard cluster on kubernetes
 * [vineyardctl deploy cert-manager](#vineyardctl-deploy-cert-manager)	 - Deploy the cert-manager on kubernetes
+* [vineyardctl deploy csidriver](#vineyardctl-deploy-csidriver)	 - Deploy the vineyard csi driver on kubernetes
 * [vineyardctl deploy operator](#vineyardctl-deploy-operator)	 - Deploy the vineyard operator on kubernetes
 * [vineyardctl deploy recover-job](#vineyardctl-deploy-recover-job)	 - Deploy a recover job to recover a backup of current vineyard cluster on kubernetes
 * [vineyardctl deploy vineyard-cluster](#vineyardctl-deploy-vineyard-cluster)	 - Deploy the vineyard cluster from kubernetes
@@ -561,6 +618,10 @@ Deploy the vineyard components on kubernetes
 
   # deploy the vineyardd on kubernetes
   vineyardctl -n vineyard-system --kubeconfig $HOME/.kube/config deploy vineyardd
+  
+  # deploy the vineyard csi driver on kubernetes
+  vineyardctl deploy csidriver --name vineyard-csi-sample \
+    --clusters vineyard-system/vineyardd-sample,default/vineyardd-sample
 ```
 
 ### Options
@@ -729,6 +790,53 @@ vineyardctl deploy cert-manager [flags]
   -h, --help   help for cert-manager
 ```
 
+## `vineyardctl deploy csidriver`
+
+Deploy the vineyard csi driver on kubernetes
+
+### Synopsis
+
+Deploy the Vineyard CSI Driver on kubernetes. 
+The CR is a cluster-scoped resource, and can only be created once.
+
+```
+vineyardctl deploy csidriver [flags]
+```
+
+**SEE ALSO**
+
+* [vineyardctl deploy](#vineyardctl-deploy)	 - Deploy the vineyard components on kubernetes
+
+### Examples
+
+```shell
+  # deploy the Vineyard CSI Driver named vineyard-csi-sample on kubernetes
+  # Notice, the clusters are built as {vineyard-deployment-namespace}/{vineyard-deployment-name}
+  # and sperated by comma, e.g. vineyard-system/vineyardd-sample, default/vineyardd-sample
+  # They must be created before deploying the Vineyard CSI Driver.
+  vineyardctl deploy csidriver --name vineyard-csi-sample \
+    --clusters vineyard-system/vineyardd-sample,default/vineyardd-sample
+```
+
+### Options
+
+```
+      --attacherImage string             The image of csi attacher. (default "registry.k8s.io/sig-storage/csi-attacher:v4.0.0")
+      --clusters strings                 The list of vineyard clusters.
+      --enableToleration                 Enable toleration for vineyard csi driver.
+  -h, --help                             help for csidriver
+  -i, --image string                     The image of vineyard csi driver. (default "vineyardcloudnative/vineyard-csi-driver")
+      --imagePullPolicy string           The image pull policy of vineyard csi driver. (default "IfNotPresent")
+      --livenessProbeImage string        The image of livenessProbe. (default "registry.k8s.io/sig-storage/livenessprobe:v2.8.0")
+      --name string                      The name of the csi driver cr. (default "csidriver-sample")
+      --nodeRegistrarImage string        The image of csi nodeRegistrar. (default "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.0")
+      --provisionerImage string          The image of csi provisioner. (default "registry.k8s.io/sig-storage/csi-provisioner:v3.3.0")
+      --sidecar.enableTopology           Enable topology for the csi driver.
+      --sidecar.imagePullPolicy string   The image pull policy of all sidecar containers. (default "Always")
+  -s, --storageClassName string          The name of storage class. (default "vineyard-csi")
+  -m, --volumeBindingMode string         The volume binding mode of the storage class. (default "WaitForFirstConsumer")
+```
+
 ## `vineyardctl deploy operator`
 
 Deploy the vineyard operator on kubernetes
@@ -2160,4 +2268,4 @@ vineyardctl schedule workload [flags]
       --resource string              the json string of kubernetes workload
       --vineyardd-name string        the namespace of vineyard cluster (default "vineyardd-sample")
       --vineyardd-namespace string   the namespace of vineyard cluster (default "vineyard-system")
-```
\ No newline at end of file
+```
diff --git a/k8s/cmd/commands/csi/csi.go b/k8s/cmd/commands/csi/csi.go
new file mode 100644
index 00000000..82e12904
--- /dev/null
+++ b/k8s/cmd/commands/csi/csi.go
@@ -0,0 +1,50 @@
+/*
+* Copyright 2020-2023 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package csi contains the start command of vineyard csi driver
+package csi
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/v6d-io/v6d/k8s/cmd/commands/flags"
+	"github.com/v6d-io/v6d/k8s/cmd/commands/util"
+	"github.com/v6d-io/v6d/k8s/pkg/csidriver"
+)
+
+var csiExample = util.Examples(`
+	# start the csi with the specific endpoint and node id
+	vineyardctl csi --endpoint=unix:///csi/csi.sock --nodeid=csinode1`)
+
+// csiCmd starts the vineyard csi driver
+var csiCmd = &cobra.Command{
+	Use:     "csi",
+	Short:   "Start the vineyard csi driver",
+	Example: csiExample,
+	Run: func(cmd *cobra.Command, args []string) {
+		util.AssertNoArgs(cmd, args)
+		d := csidriver.NewDriver(flags.NodeID, flags.Endpoint)
+		d.Run()
+	},
+}
+
+func NewCsiCmd() *cobra.Command {
+	return csiCmd
+}
+
+func init() {
+	flags.ApplyCsiOpts(csiCmd)
+}
diff --git a/k8s/cmd/commands/csi/csi_test.go b/k8s/cmd/commands/csi/csi_test.go
new file mode 100644
index 00000000..68cfb30b
--- /dev/null
+++ b/k8s/cmd/commands/csi/csi_test.go
@@ -0,0 +1,67 @@
+/*
+* Copyright 2020-2023 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package csi contains the start command of vineyard csi driver
+package csi
+
+import (
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+	"testing"
+
+	"github.com/kubernetes-csi/csi-test/v4/pkg/sanity"
+	"github.com/v6d-io/v6d/k8s/cmd/commands/flags"
+	"github.com/v6d-io/v6d/k8s/pkg/csidriver"
+)
+
+func TestVineyardCSIDriver(t *testing.T) {
+	// Setup the full driver and its environment
+	csiSocket := "/tmp/csi.sock"
+	csiEndpoint := "unix://" + csiSocket
+	if err := os.Remove(csiSocket); err != nil && !os.IsNotExist(err) {
+		t.Errorf("failed to remove socket file %s: %v", csiSocket, err)
+		os.Exit(1)
+	}
+
+	flags.Endpoint = csiEndpoint
+	flags.NodeID = "test-node-id"
+
+	vineyardSocket := filepath.Join(csidriver.VineyardSocketPrefix, csidriver.VineyardSocket)
+	if _, err := os.OpenFile(vineyardSocket, os.O_CREATE|os.O_RDONLY, 0666); err != nil {
+		t.Errorf("failed to open vineyard socket file %s: %v", vineyardSocket, err)
+	}
+
+	// Create a channel to signal the goroutine to stop
+	stop := make(chan os.Signal, 1)
+	signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		csiCmd.Run(csiCmd, []string{})
+	}()
+	config := sanity.NewTestConfig()
+	config.Address = csiEndpoint
+	config.TargetPath = "/opt/target"
+	config.StagingPath = "/opt/staging"
+
+	sanity.Test(t, config)
+
+	// Wait for a stop signal
+	<-stop
+
+	// Exit the program with a status code of 0 (success)
+	os.Exit(0)
+}
diff --git a/k8s/cmd/commands/delete/delete.go b/k8s/cmd/commands/delete/delete.go
index 20caa628..9f3eaa71 100644
--- a/k8s/cmd/commands/delete/delete.go
+++ b/k8s/cmd/commands/delete/delete.go
@@ -55,4 +55,5 @@ func init() {
 	deleteCmd.AddCommand(NewDeleteRecoverCmd())
 	deleteCmd.AddCommand(NewDeleteVineyardDeploymentCmd())
 	deleteCmd.AddCommand(NewDeleteOperationCmd())
+	deleteCmd.AddCommand(NewDeleteCSIDriverCmd())
 }
diff --git a/k8s/cmd/commands/delete/delete_csidriver.go b/k8s/cmd/commands/delete/delete_csidriver.go
new file mode 100644
index 00000000..aee2753c
--- /dev/null
+++ b/k8s/cmd/commands/delete/delete_csidriver.go
@@ -0,0 +1,61 @@
+/*
+* Copyright 2020-2023 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package delete
+
+import (
+	"github.com/spf13/cobra"
+	"k8s.io/apimachinery/pkg/types"
+
+	"github.com/v6d-io/v6d/k8s/apis/k8s/v1alpha1"
+	"github.com/v6d-io/v6d/k8s/cmd/commands/flags"
+	"github.com/v6d-io/v6d/k8s/cmd/commands/util"
+	"github.com/v6d-io/v6d/k8s/pkg/log"
+)
+
+var deleteCSIDriverExample = util.Examples(`
+	# delete the csi driver named "csidriver-test"
+	vineyardctl delete csidriver --name csidriver-test`)
+
+// deleteCSIDriverCmd deletes the specific operation
+var deleteCSIDriverCmd = &cobra.Command{
+	Use:     "csidriver",
+	Short:   "Delete the vineyard csi driver on kubernetes",
+	Example: deleteCSIDriverExample,
+	Run: func(cmd *cobra.Command, args []string) {
+		util.AssertNoArgs(cmd, args)
+
+		client := util.KubernetesClient()
+
+		log.Info("deleting CSIDriver cr")
+		csiDriver := &v1alpha1.CSIDriver{}
+
+		if err := util.Delete(client, types.NamespacedName{
+			Name: flags.CSIDriverName,
+		}, csiDriver); err != nil {
+			log.Fatal(err, "failed to delete CSIDriver")
+		}
+
+		log.Info("CSIDriver is deleted.")
+	},
+}
+
+func NewDeleteCSIDriverCmd() *cobra.Command {
+	return deleteCSIDriverCmd
+}
+
+func init() {
+	flags.ApplyCSIDriverNameOpts(deleteCSIDriverCmd)
+}
diff --git a/k8s/cmd/commands/deploy/deploy.go b/k8s/cmd/commands/deploy/deploy.go
index 32816482..580e096a 100644
--- a/k8s/cmd/commands/deploy/deploy.go
+++ b/k8s/cmd/commands/deploy/deploy.go
@@ -32,7 +32,11 @@ var deployExample = util.Examples(`
 	vineyardctl -n vineyard-system --kubeconfig $HOME/.kube/config deploy cert-manager
 
 	# deploy the vineyardd on kubernetes
-	vineyardctl -n vineyard-system --kubeconfig $HOME/.kube/config deploy vineyardd`)
+	vineyardctl -n vineyard-system --kubeconfig $HOME/.kube/config deploy vineyardd
+	
+	# deploy the vineyard csi driver on kubernetes
+	vineyardctl deploy csidriver --name vineyard-csi-sample \
+		--clusters vineyard-system/vineyardd-sample,default/vineyardd-sample`)
 
 // deployCmd deploys all vineyard components on kubernetes
 var deployCmd = &cobra.Command{
@@ -53,4 +57,5 @@ func init() {
 	deployCmd.AddCommand(NewDeployVineyardDeploymentCmd())
 	deployCmd.AddCommand(NewDeployBackupJobCmd())
 	deployCmd.AddCommand(NewDeployRecoverJobCmd())
+	deployCmd.AddCommand(NewDeployCSIDriverCmd())
 }
diff --git a/k8s/cmd/commands/deploy/deploy_csidriver.go b/k8s/cmd/commands/deploy/deploy_csidriver.go
new file mode 100644
index 00000000..0c995e9f
--- /dev/null
+++ b/k8s/cmd/commands/deploy/deploy_csidriver.go
@@ -0,0 +1,107 @@
+/*
+* Copyright 2020-2023 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package deploy
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/v6d-io/v6d/k8s/apis/k8s/v1alpha1"
+	"github.com/v6d-io/v6d/k8s/cmd/commands/flags"
+	"github.com/v6d-io/v6d/k8s/cmd/commands/util"
+	"github.com/v6d-io/v6d/k8s/pkg/log"
+)
+
+var (
+	deployCSIDriverLong = util.LongDesc(`
+	Deploy the Vineyard CSI Driver on kubernetes. 
+	The CR is a cluster-scoped resource, and can only be created once.`)
+
+	deployCSIDriverExample = util.Examples(`
+	# deploy the Vineyard CSI Driver named vineyard-csi-sample on kubernetes
+	# Notice, the clusters are built as {vineyard-deployment-namespace}/{vineyard-deployment-name}
+	# and sperated by comma, e.g. vineyard-system/vineyardd-sample, default/vineyardd-sample
+	# They must be created before deploying the Vineyard CSI Driver.
+	vineyardctl deploy csidriver --name vineyard-csi-sample \
+		--clusters vineyard-system/vineyardd-sample,default/vineyardd-sample`)
+)
+
+// deployCSIDriverCmd deploys the vineyard csi driver on kubernetes
+var deployCSIDriverCmd = &cobra.Command{
+	Use:     "csidriver",
+	Short:   "Deploy the vineyard csi driver on kubernetes",
+	Long:    deployCSIDriverLong,
+	Example: deployCSIDriverExample,
+	Run: func(cmd *cobra.Command, args []string) {
+		util.AssertNoArgsOrInput(cmd, args)
+
+		client := util.KubernetesClient()
+		log.Info("building CSIDriver cr")
+		csiDriver, err := BuildCSIDriver()
+		if err != nil {
+			log.Fatal(err, "Failed to build csi driver")
+		}
+		log.Info("creating csi driver")
+		if err := util.Create(client, csiDriver, func(csiDriver *v1alpha1.CSIDriver) bool {
+			return csiDriver.Status.State == v1alpha1.CSIDriverRunning
+		}); err != nil {
+			log.Fatal(err, "failed to create/wait Vineyard CSI Driver")
+		}
+
+		log.Info("Vineyard CSI Driver is ready.")
+	},
+}
+
+func checkVolumeBindingMode(mode string) bool {
+	switch mode {
+	case "WaitForFirstConsumer", "Immediate":
+		return true
+	default:
+		return false
+	}
+}
+
+func BuildCSIDriver() (*v1alpha1.CSIDriver, error) {
+	clusters, err := util.ParseVineyardClusters(flags.VineyardClusters)
+	if err != nil {
+		return nil, err
+	}
+	csiDriver := &v1alpha1.CSIDriver{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      flags.CSIDriverName,
+			Namespace: flags.GetDefaultVineyardNamespace(),
+		},
+		Spec: flags.CSIDriverOpts,
+	}
+	if !checkVolumeBindingMode(csiDriver.Spec.VolumeBindingMode) {
+		return nil, fmt.Errorf("invalid volume binding mode: %s, "+
+			"only support WaitForFirstConsumer and Immediate",
+			csiDriver.Spec.VolumeBindingMode)
+	}
+	csiDriver.Spec.Clusters = *clusters
+	csiDriver.Spec.EnableVerboseLog = flags.Verbose
+	return csiDriver, nil
+}
+
+func NewDeployCSIDriverCmd() *cobra.Command {
+	return deployCSIDriverCmd
+}
+
+func init() {
+	flags.ApplyCSIDriverOpts(deployCSIDriverCmd)
+}
diff --git a/k8s/cmd/commands/flags/csi_flags.go b/k8s/cmd/commands/flags/csi_flags.go
new file mode 100644
index 00000000..777715cd
--- /dev/null
+++ b/k8s/cmd/commands/flags/csi_flags.go
@@ -0,0 +1,38 @@
+/*
+* Copyright 2020-2023 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package flags
+
+import "github.com/spf13/cobra"
+
+var (
+	// Endpoint is the endpoint of vineyard csi driver
+	Endpoint string
+
+	// NodeID is the node id of vineyard csi driver
+	NodeID string
+
+	// StateFilePath is the path of state file
+	StateFilePath string
+)
+
+func ApplyCsiOpts(cmd *cobra.Command) {
+	cmd.Flags().StringVarP(&Endpoint, "endpoint", "f", "",
+		"the endpoint of vineyard csi driver")
+	cmd.Flags().StringVarP(&NodeID, "nodeid", "", "",
+		"the node id of vineyard csi driver")
+	cmd.Flags().StringVarP(&StateFilePath, "state-file-path", "", "/csi/state",
+		"the path of state file")
+}
diff --git a/k8s/cmd/commands/flags/csidriver_flags.go b/k8s/cmd/commands/flags/csidriver_flags.go
new file mode 100644
index 00000000..5b76fb7b
--- /dev/null
+++ b/k8s/cmd/commands/flags/csidriver_flags.go
@@ -0,0 +1,73 @@
+/*
+* Copyright 2020-2023 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package flags
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/v6d-io/v6d/k8s/apis/k8s/v1alpha1"
+)
+
+var (
+	// CSIDriverOpts holds all configuration of CSIDriver Spec
+	CSIDriverOpts v1alpha1.CSIDriverSpec
+
+	// CSIDriverName is the name of the csi driver cr
+	CSIDriverName string
+
+	// VineyardClusters contains all the vineyard clusters
+	VineyardClusters []string
+)
+
+func ApplyCSIDriverNameOpts(cmd *cobra.Command) {
+	cmd.Flags().StringVarP(&CSIDriverName, "name", "", "csidriver-sample",
+		"The name of the csi driver cr.")
+}
+func ApplyCSIDriverSidecarOpts(cmd *cobra.Command) {
+	cmd.Flags().StringVarP(&CSIDriverOpts.Sidecar.ProvisionerImage, "provisionerImage", "",
+		"registry.k8s.io/sig-storage/csi-provisioner:v3.3.0", "The image of csi provisioner.")
+	cmd.Flags().StringVarP(&CSIDriverOpts.Sidecar.AttacherImage, "attacherImage", "",
+		"registry.k8s.io/sig-storage/csi-attacher:v4.0.0", "The image of csi attacher.")
+	cmd.Flags().StringVarP(&CSIDriverOpts.Sidecar.NodeRegistrarImage, "nodeRegistrarImage", "",
+		"registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.0", "The image of csi nodeRegistrar.")
+	cmd.Flags().StringVarP(&CSIDriverOpts.Sidecar.LivenessProbeImage, "livenessProbeImage", "",
+		"registry.k8s.io/sig-storage/livenessprobe:v2.8.0", "The image of livenessProbe.")
+	cmd.Flags().StringVarP(&CSIDriverOpts.Sidecar.ImagePullPolicy, "sidecar.imagePullPolicy", "",
+		"Always", "The image pull policy of all sidecar containers.")
+	cmd.Flags().BoolVarP(&CSIDriverOpts.Sidecar.EnableTopology, "sidecar.enableTopology", "",
+		false, "Enable topology for the csi driver.")
+}
+
+func ApplyCSIDriverClustersOpts(cmd *cobra.Command) {
+	cmd.Flags().StringSliceVarP(&VineyardClusters, "clusters", "",
+		[]string{}, "The list of vineyard clusters.")
+}
+
+func ApplyCSIDriverOpts(cmd *cobra.Command) {
+	ApplyCSIDriverNameOpts(cmd)
+	cmd.Flags().StringVarP(&CSIDriverOpts.Image, "image", "i",
+		"vineyardcloudnative/vineyard-csi-driver",
+		"The image of vineyard csi driver.")
+	cmd.Flags().StringVarP(&CSIDriverOpts.ImagePullPolicy, "imagePullPolicy", "",
+		"IfNotPresent", "The image pull policy of vineyard csi driver.")
+	cmd.Flags().StringVarP(&CSIDriverOpts.StorageClassName, "storageClassName", "s",
+		"vineyard-csi", "The name of storage class.")
+	cmd.Flags().StringVarP(&CSIDriverOpts.VolumeBindingMode, "volumeBindingMode", "m",
+		"WaitForFirstConsumer", "The volume binding mode of the storage class.")
+	cmd.Flags().BoolVarP(&CSIDriverOpts.EnableToleration, "enableToleration", "",
+		false, "Enable toleration for vineyard csi driver.")
+	ApplyCSIDriverSidecarOpts(cmd)
+	ApplyCSIDriverClustersOpts(cmd)
+}
diff --git a/k8s/cmd/commands/manager/manager.go b/k8s/cmd/commands/manager/manager.go
index 94054d7d..b1ff343a 100644
--- a/k8s/cmd/commands/manager/manager.go
+++ b/k8s/cmd/commands/manager/manager.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// Package start contains the start command of vineyard operator
+// Package manager contains the start command of vineyard operator
 package manager
 
 import (
@@ -164,6 +164,14 @@ func startManager(
 		log.Fatal(err, "unable to create recover controller")
 	}
 
+	if err := (&controllers.CSIDriverReconciler{
+		Client:        mgr.GetClient(),
+		Scheme:        mgr.GetScheme(),
+		EventRecorder: mgr.GetEventRecorderFor("csidriver-controller"),
+	}).SetupWithManager(mgr); err != nil {
+		log.Fatal(err, "unable to create csidriver controller")
+	}
+
 	if flags.EnableWebhook {
 		// register the webhooks of CRDs
 		if err := (&v1alpha1.LocalObject{}).SetupWebhookWithManager(mgr); err != nil {
@@ -187,6 +195,9 @@ func startManager(
 		if err := (&v1alpha1.Recover{}).SetupWebhookWithManager(mgr); err != nil {
 			log.Fatal(err, "unable to create recover webhook")
 		}
+		if err := (&v1alpha1.CSIDriver{}).SetupWebhookWithManager(mgr); err != nil {
+			log.Fatal(err, "unable to create csidriver webhook")
+		}
 
 		// register the assembly webhook
 		log.Info("registering the assembly webhook")
diff --git a/k8s/cmd/commands/util/parse.go b/k8s/cmd/commands/util/parse.go
index 33aee0c9..200f27ce 100644
--- a/k8s/cmd/commands/util/parse.go
+++ b/k8s/cmd/commands/util/parse.go
@@ -17,9 +17,11 @@ package util
 
 import (
 	"encoding/json"
+	"fmt"
 	"strings"
 
 	"github.com/ghodss/yaml"
+	"github.com/v6d-io/v6d/k8s/apis/k8s/v1alpha1"
 
 	"github.com/pkg/errors"
 
@@ -180,3 +182,22 @@ func ParseOwnerRef(of string) ([]metav1.OwnerReference, error) {
 
 	return ownerRef, nil
 }
+
+//	ParseVineyardClusters parse the []string to nested []{
+//		"namespace": "vineyard-cluster-namespace",
+//	    "name": "vineyard-cluster-name",
+//	}
+func ParseVineyardClusters(clusters []string) (*[]v1alpha1.VineyardClusters, error) {
+	vineyardClusters := make([]v1alpha1.VineyardClusters, 0)
+	for i := range clusters {
+		s := strings.Split(clusters[i], "/")
+		if len(s) != 2 {
+			return nil, errors.Wrap(fmt.Errorf("invalid vineyard cluster %s", clusters[i]), "parse vineyard cluster")
+		}
+		vineyardClusters = append(vineyardClusters, v1alpha1.VineyardClusters{
+			Namespace: s[0],
+			Name:      s[1],
+		})
+	}
+	return &vineyardClusters, nil
+}
diff --git a/k8s/cmd/main.go b/k8s/cmd/main.go
index 538003d9..e291d558 100644
--- a/k8s/cmd/main.go
+++ b/k8s/cmd/main.go
@@ -27,6 +27,7 @@ import (
 	gosdklog "github.com/v6d-io/v6d/go/vineyard/pkg/common/log"
 	"github.com/v6d-io/v6d/k8s/cmd/commands/client"
 	"github.com/v6d-io/v6d/k8s/cmd/commands/create"
+	"github.com/v6d-io/v6d/k8s/cmd/commands/csi"
 	"github.com/v6d-io/v6d/k8s/cmd/commands/delete"
 	"github.com/v6d-io/v6d/k8s/cmd/commands/deploy"
 	"github.com/v6d-io/v6d/k8s/cmd/commands/flags"
@@ -74,6 +75,7 @@ func init() {
 	cmd.AddCommand(client.NewLsCmd())
 	cmd.AddCommand(client.NewGetCmd())
 	cmd.AddCommand(client.NewPutCmd())
+	cmd.AddCommand(csi.NewCsiCmd())
 }
 
 func main() {
diff --git a/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml b/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml
new file mode 100644
index 00000000..c74d2cc3
--- /dev/null
+++ b/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml
@@ -0,0 +1,97 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.11.0
+  creationTimestamp: null
+  name: csidrivers.k8s.v6d.io
+spec:
+  group: k8s.v6d.io
+  names:
+    kind: CSIDriver
+    listKind: CSIDriverList
+    plural: csidrivers
+    singular: csidriver
+  scope: Cluster
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        properties:
+          apiVersion:
+            type: string
+          kind:
+            type: string
+          metadata:
+            type: object
+          spec:
+            properties:
+              clusters:
+                items:
+                  properties:
+                    name:
+                      default: ""
+                      type: string
+                    namespace:
+                      default: ""
+                      type: string
+                  type: object
+                type: array
+              enableToleration:
+                default: false
+                type: boolean
+              enableVerboseLog:
+                default: false
+                type: boolean
+              image:
+                default: vineyardcloudnative/vineyard-operator
+                type: string
+              imagePullPolicy:
+                default: IfNotPresent
+                type: string
+              sidecar:
+                default:
+                  attacherImage: registry.k8s.io/sig-storage/csi-attacher:v4.0.0
+                  enableTopology: false
+                  imagePullPolicy: Always
+                  livenessProbeImage: registry.k8s.io/sig-storage/livenessprobe:v2.8.0
+                  nodeRegistrarImage: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.0
+                  provisionerImage: registry.k8s.io/sig-storage/csi-provisioner:v3.3.0
+                properties:
+                  attacherImage:
+                    default: registry.k8s.io/sig-storage/csi-attacher:v4.0.0
+                    type: string
+                  enableTopology:
+                    default: false
+                    type: boolean
+                  imagePullPolicy:
+                    default: Always
+                    type: string
+                  livenessProbeImage:
+                    default: registry.k8s.io/sig-storage/livenessprobe:v2.8.0
+                    type: string
+                  nodeRegistrarImage:
+                    default: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.0
+                    type: string
+                  provisionerImage:
+                    default: registry.k8s.io/sig-storage/csi-provisioner:v3.3.0
+                    type: string
+                type: object
+              storageClassName:
+                default: vineyard-csi
+                type: string
+              volumeBindingMode:
+                default: WaitForFirstConsumer
+                type: string
+            type: object
+          status:
+            properties:
+              state:
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
diff --git a/k8s/config/crd/kustomization.yaml b/k8s/config/crd/kustomization.yaml
index 282bd62f..dda9bc5f 100644
--- a/k8s/config/crd/kustomization.yaml
+++ b/k8s/config/crd/kustomization.yaml
@@ -9,6 +9,7 @@ resources:
 - bases/k8s.v6d.io_sidecars.yaml
 - bases/k8s.v6d.io_backups.yaml
 - bases/k8s.v6d.io_recovers.yaml
+- bases/k8s.v6d.io_csidrivers.yaml
 # +kubebuilder:scaffold:crdkustomizeresource
 
 patchesStrategicMerge:
@@ -20,7 +21,8 @@ patchesStrategicMerge:
 - patches/webhook_in_operations.yaml
 - patches/webhook_in_sidecars.yaml
 - patches/webhook_in_backups.yaml
-#- patches/webhook_in_recovers.yaml
+- patches/webhook_in_recovers.yaml
+- patches/webhook_in_csidrivers.yaml
 # +kubebuilder:scaffold:crdkustomizewebhookpatch
 
 # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix.
@@ -31,7 +33,8 @@ patchesStrategicMerge:
 - patches/cainjection_in_operations.yaml
 - patches/cainjection_in_sidecars.yaml
 - patches/cainjection_in_backups.yaml
-#- patches/cainjection_in_recovers.yaml
+- patches/cainjection_in_recovers.yaml
+- patches/cainjection_in_csidrivers.yaml
 # +kubebuilder:scaffold:crdkustomizecainjectionpatch
 
 # the following config is for teaching kustomize how to do kustomization for CRDs.
diff --git a/k8s/config/crd/patches/cainjection_in_csidrivers.yaml b/k8s/config/crd/patches/cainjection_in_csidrivers.yaml
new file mode 100644
index 00000000..6e5108bb
--- /dev/null
+++ b/k8s/config/crd/patches/cainjection_in_csidrivers.yaml
@@ -0,0 +1,7 @@
+# The following patch adds a directive for certmanager to inject CA into the CRD
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
+  name: csidrivers.k8s.v6d.io
diff --git a/k8s/config/crd/patches/cainjection_in_k8s_recovers.yaml b/k8s/config/crd/patches/cainjection_in_recovers.yaml
similarity index 100%
rename from k8s/config/crd/patches/cainjection_in_k8s_recovers.yaml
rename to k8s/config/crd/patches/cainjection_in_recovers.yaml
diff --git a/k8s/config/crd/patches/webhook_in_csidrivers.yaml b/k8s/config/crd/patches/webhook_in_csidrivers.yaml
new file mode 100644
index 00000000..a30d15ea
--- /dev/null
+++ b/k8s/config/crd/patches/webhook_in_csidrivers.yaml
@@ -0,0 +1,16 @@
+# The following patch enables a conversion webhook for the CRD
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: csidrivers.k8s.v6d.io
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          namespace: system
+          name: webhook-service
+          path: /convert
+      conversionReviewVersions:
+      - v1
diff --git a/k8s/config/crd/patches/webhook_in_k8s_recovers.yaml b/k8s/config/crd/patches/webhook_in_recovers.yaml
similarity index 100%
rename from k8s/config/crd/patches/webhook_in_k8s_recovers.yaml
rename to k8s/config/crd/patches/webhook_in_recovers.yaml
diff --git a/k8s/config/manager/manager.yaml b/k8s/config/manager/manager.yaml
index 519cf333..49e486b7 100644
--- a/k8s/config/manager/manager.yaml
+++ b/k8s/config/manager/manager.yaml
@@ -11,6 +11,7 @@ metadata:
   name: controller-manager
   namespace: system
   labels:
+    k8s.v6d.io/instance: vineyard-operator
     control-plane: controller-manager
 spec:
   selector:
diff --git a/k8s/config/rbac/k8s_csidriver_editor_role.yaml b/k8s/config/rbac/k8s_csidriver_editor_role.yaml
new file mode 100644
index 00000000..a98ac668
--- /dev/null
+++ b/k8s/config/rbac/k8s_csidriver_editor_role.yaml
@@ -0,0 +1,24 @@
+# permissions for end users to edit csidrivers.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: csidriver-editor-role
+rules:
+- apiGroups:
+  - k8s.v6d.io
+  resources:
+  - csidrivers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - k8s.v6d.io
+  resources:
+  - csidrivers/status
+  verbs:
+  - get
diff --git a/k8s/config/rbac/k8s_csidriver_viewer_role.yaml b/k8s/config/rbac/k8s_csidriver_viewer_role.yaml
new file mode 100644
index 00000000..c8aa1c70
--- /dev/null
+++ b/k8s/config/rbac/k8s_csidriver_viewer_role.yaml
@@ -0,0 +1,20 @@
+# permissions for end users to view csidrivers.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: csidriver-viewer-role
+rules:
+- apiGroups:
+  - k8s.v6d.io
+  resources:
+  - csidrivers
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - k8s.v6d.io
+  resources:
+  - csidrivers/status
+  verbs:
+  - get
diff --git a/k8s/config/rbac/manager_account.yaml b/k8s/config/rbac/manager_account.yaml
index 9f54994d..e1f683cf 100644
--- a/k8s/config/rbac/manager_account.yaml
+++ b/k8s/config/rbac/manager_account.yaml
@@ -1,10 +1,8 @@
-
 ---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: manager
-
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -17,7 +15,6 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: manager
-
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -30,7 +27,6 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: manager
-
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -43,8 +39,6 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: manager
-
-
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
diff --git a/k8s/config/rbac/role.yaml b/k8s/config/rbac/role.yaml
index 90efe900..97eba3d4 100644
--- a/k8s/config/rbac/role.yaml
+++ b/k8s/config/rbac/role.yaml
@@ -23,6 +23,14 @@ rules:
   verbs:
   - create
   - patch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - ""
   resources:
@@ -34,6 +42,13 @@ rules:
   - list
   - patch
   - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims/finalizers
+  verbs:
+  - patch
 - apiGroups:
   - ""
   resources:
@@ -45,6 +60,13 @@ rules:
   - list
   - patch
   - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes/finalizers
+  verbs:
+  - patch
 - apiGroups:
   - ""
   resources:
@@ -90,6 +112,16 @@ rules:
   - delete
   - get
   - update
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - create
+  - get
+  - list
+  - update
+  - watch
 - apiGroups:
   - apps
   resources:
@@ -127,6 +159,14 @@ rules:
   - get
   - list
   - update
+- apiGroups:
+  - csi.storage.k8s.io
+  resources:
+  - csinodeinfos
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - k8s.v6d.io
   resources:
@@ -147,6 +187,32 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - k8s.v6d.io
+  resources:
+  - csidrivers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - k8s.v6d.io
+  resources:
+  - csidrivers/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - k8s.v6d.io
+  resources:
+  - csidrivers/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - k8s.v6d.io
   resources:
@@ -285,3 +351,71 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - update
+  - watch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshots
+  verbs:
+  - get
+  - list
+  - update
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - csinodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - storageclasses
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments/status
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
diff --git a/k8s/config/samples/k8s_v1alpha1_csidriver.yaml b/k8s/config/samples/k8s_v1alpha1_csidriver.yaml
new file mode 100644
index 00000000..e344eb85
--- /dev/null
+++ b/k8s/config/samples/k8s_v1alpha1_csidriver.yaml
@@ -0,0 +1,10 @@
+apiVersion: k8s.v6d.io/v1alpha1
+kind: CSIDriver
+metadata:
+  name: csidriver-sample
+spec:
+  clusters:
+  - namespace: vineyard-system
+    name: vineyardd-sample
+  - namespace: default
+    name: vineyardd-sample
\ No newline at end of file
diff --git a/k8s/config/webhook/manifests.yaml b/k8s/config/webhook/manifests.yaml
index ff88f70e..f815a52b 100644
--- a/k8s/config/webhook/manifests.yaml
+++ b/k8s/config/webhook/manifests.yaml
@@ -231,6 +231,26 @@ webhooks:
     resources:
     - backups
   sideEffects: None
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-k8s-v6d-io-v1alpha1-csidriver
+  failurePolicy: Fail
+  name: vcsidriver.kb.io
+  rules:
+  - apiGroups:
+    - k8s.v6d.io
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - csidrivers
+  sideEffects: None
 - admissionReviewVersions:
   - v1
   clientConfig:
diff --git a/k8s/controllers/k8s/csidriver_controller.go b/k8s/controllers/k8s/csidriver_controller.go
new file mode 100644
index 00000000..ef6d57cb
--- /dev/null
+++ b/k8s/controllers/k8s/csidriver_controller.go
@@ -0,0 +1,219 @@
+/** Copyright 2020-2023 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package k8s
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	appsv1 "k8s.io/api/apps/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/apache/skywalking-swck/operator/pkg/kubernetes"
+	"github.com/pkg/errors"
+
+	k8sv1alpha1 "github.com/v6d-io/v6d/k8s/apis/k8s/v1alpha1"
+	v1alpha1 "github.com/v6d-io/v6d/k8s/apis/k8s/v1alpha1"
+	"github.com/v6d-io/v6d/k8s/pkg/log"
+	"github.com/v6d-io/v6d/k8s/pkg/templates"
+)
+
+// CSIDriverReconciler reconciles a CSIDriver object
+type CSIDriverReconciler struct {
+	client.Client
+	record.EventRecorder
+	Scheme *runtime.Scheme
+}
+
+type StorageConfig struct {
+	Namespace         string
+	Name              string
+	VolumeBindingMode string
+}
+
+//+kubebuilder:rbac:groups=k8s.v6d.io,resources=csidrivers,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=k8s.v6d.io,resources=csidrivers/status,verbs=get;update;patch
+//+kubebuilder:rbac:groups=k8s.v6d.io,resources=csidrivers/finalizers,verbs=update
+//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update
+//+kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=get;list;watch;create;update
+//+kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch
+//+kubebuilder:rbac:groups="",resources=persistentvolumes,verbs=get;list;watch;update;patch;create;delete
+//+kubebuilder:rbac:groups="",resources=persistentvolumes/finalizers,verbs=patch
+//+kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;update
+//+kubebuilder:rbac:groups="",resources=persistentvolumeclaims/finalizers,verbs=patch
+//+kubebuilder:rbac:groups=storage.k8s.io, resources=storageclasses, verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=storage.k8s.io, resources=csinodes, verbs=get;list;watch
+//+kubebuilder:rbac:groups=storage.k8s.io,resources=volumeattachments,verbs=get;list;watch;update;patch
+//+kubebuilder:rbac:groups=storage.k8s.io,resources=volumeattachments/status,verbs=get;list;watch;update;patch
+//+kubebuilder:rbac:groups=csi.storage.k8s.io,resources=csinodeinfos,verbs=get;list;watch
+//+kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshotclasses,verbs=get;list;watch
+//+kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshotcontents,verbs=get;list;watch;create;update;delete
+//+kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshots,verbs=get;list;watch;update
+
+func (r *CSIDriverReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	logger := log.FromContext(ctx).WithName("controllers").WithName("CSIDriver")
+
+	csiDriver := &k8sv1alpha1.CSIDriver{}
+	if err := r.Client.Get(ctx, req.NamespacedName, csiDriver); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	logger.Info("Reconciling CSIDriver", "csiDriver", csiDriver)
+
+	// check there is no more than one csi driver in the cluster
+	csidrivers := &k8sv1alpha1.CSIDriverList{}
+	if err := r.Client.List(ctx, csidrivers); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	if len(csidrivers.Items) > 1 {
+		logger.Error(nil, "There is already a csi driver in the cluster")
+		return ctrl.Result{}, nil
+	}
+
+	// check the vineyard clusters are ready
+	if len(csiDriver.Spec.Clusters) == 0 {
+		logger.Error(nil, "No vineyard cluster is specified")
+		return ctrl.Result{}, nil
+	}
+	deployment := &appsv1.Deployment{}
+	for _, c := range csiDriver.Spec.Clusters {
+		if err := r.Client.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: c.Name}, deployment); err != nil {
+			return ctrl.Result{}, err
+		}
+		if deployment.Status.ReadyReplicas != *deployment.Spec.Replicas {
+			logger.Error(nil, fmt.Sprintf("Vineyard deployment %s/%s is not ready", c.Namespace, c.Name))
+			return ctrl.Result{}, nil
+		}
+	}
+
+	// get the namespace of the vineyard operator
+	deploymentLists := &appsv1.DeploymentList{}
+	if err := r.Client.List(ctx, deploymentLists, client.MatchingLabels{"k8s.v6d.io/instance": "vineyard-operator"}); err != nil {
+		return ctrl.Result{}, err
+	}
+	if len(deploymentLists.Items) != 1 {
+		log.Errorf(nil, "Only one vineyard operator is allowed in the specific namespace, but got %v", len(deploymentLists.Items))
+		return ctrl.Result{}, nil
+	}
+	if deploymentLists.Items[0].Status.ReadyReplicas != *deploymentLists.Items[0].Spec.Replicas {
+		log.Errorf(nil, "Vineyard operator is not ready")
+		return ctrl.Result{}, nil
+	}
+
+	// the csi driver should be in the same namespace as the vineyard operator
+	// for sharing the same service account
+	csiDriver.Namespace = deploymentLists.Items[0].Namespace
+	// create a csi driver
+	csiDriverApp := kubernetes.Application{
+		Client:   r.Client,
+		CR:       csiDriver,
+		FileRepo: templates.Repo,
+		GVK:      k8sv1alpha1.GroupVersion.WithKind("CSIDriver"),
+		Recorder: r.EventRecorder,
+		TmplFunc: map[string]interface{}{},
+	}
+	if _, err := csiDriverApp.Apply(ctx, "csidriver/daemonset.yaml", logger, true); err != nil {
+		logger.Error(err, "failed to apply csidriver daemonset manifest")
+		return ctrl.Result{}, err
+	}
+	if _, err := csiDriverApp.Apply(ctx, "csidriver/deployment.yaml", logger, true); err != nil {
+		logger.Error(err, "failed to apply csidriver deployment manifest")
+		return ctrl.Result{}, err
+	}
+	for i := 0; i < len(csiDriver.Spec.Clusters); i++ {
+		csiDriverApp.TmplFunc["getStorageConfig"] = func() StorageConfig {
+			return StorageConfig{
+				Namespace:         csiDriver.Spec.Clusters[i].Namespace,
+				Name:              csiDriver.Spec.Clusters[i].Name,
+				VolumeBindingMode: csiDriver.Spec.VolumeBindingMode,
+			}
+		}
+
+		if _, err := csiDriverApp.Apply(ctx, "csidriver/storageclass.yaml", logger, true); err != nil {
+			logger.Error(err, "failed to apply csidriver manifests")
+			return ctrl.Result{}, err
+		}
+	}
+
+	if err := r.UpdateStatus(ctx, csiDriver); err != nil {
+		logger.Error(err, "Failed to update the status", "CSIDriver", csiDriver)
+	}
+	// reconcile every minute
+	duration, _ := time.ParseDuration("1m")
+	return ctrl.Result{RequeueAfter: duration}, nil
+}
+
+func (r *CSIDriverReconciler) UpdateStatus(ctx context.Context, csiDriver *k8sv1alpha1.CSIDriver) error {
+	depOK := false
+	daeOK := false
+	depName := csiDriver.Name + "-csi-driver"
+	ns := csiDriver.Namespace
+	// check if the csi driver deployment is ready
+	dep := &appsv1.Deployment{}
+	if err := r.Client.Get(ctx, types.NamespacedName{Namespace: ns, Name: depName}, dep); err != nil {
+		return errors.Wrap(err, "failed to get csi driver deployment")
+	}
+	if dep.Status.ReadyReplicas == *dep.Spec.Replicas {
+		depOK = true
+	}
+
+	// check if the csi nodes daemonset is ready
+	daeName := csiDriver.Name + "-csi-nodes"
+	dae := &appsv1.DaemonSet{}
+	if err := r.Client.Get(ctx, types.NamespacedName{Namespace: ns, Name: daeName}, dae); err != nil {
+		return errors.Wrap(err, "failed to get csi driver daemonset")
+	}
+	if dae.Status.NumberReady == dae.Status.DesiredNumberScheduled {
+		daeOK = true
+	}
+
+	state := k8sv1alpha1.CSIDriverRunning
+	if !depOK || !daeOK {
+		state = k8sv1alpha1.CSIDriverPending
+	}
+
+	status := &k8sv1alpha1.CSIDriverStatus{
+		State: state,
+	}
+	if err := ApplyStatueUpdate(ctx, r.Client, csiDriver, r.Status(),
+		func(c *k8sv1alpha1.CSIDriver) (error, *k8sv1alpha1.CSIDriver) {
+			csiDriver.Status = *status
+			csiDriver.Kind = "CSIDriver"
+
+			if err := kubernetes.ApplyOverlay(csiDriver, &v1alpha1.CSIDriver{Status: *status}); err != nil {
+				return errors.Wrap(err, "failed to overlay csidriver's status"), nil
+			}
+			return nil, csiDriver
+		},
+	); err != nil {
+		return errors.Wrap(err, "failed to update status")
+	}
+
+	return nil
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (r *CSIDriverReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&k8sv1alpha1.CSIDriver{}).
+		Complete(r)
+}
diff --git a/k8s/examples/vineyard-csidriver/Dockerfile b/k8s/examples/vineyard-csidriver/Dockerfile
new file mode 100644
index 00000000..728442cc
--- /dev/null
+++ b/k8s/examples/vineyard-csidriver/Dockerfile
@@ -0,0 +1,12 @@
+FROM python:3.10
+
+RUN pip3 install --no-cache-dir pandas requests scikit-learn numpy vineyard
+
+WORKDIR /
+
+ARG APP
+ENV APP ${APP}
+
+COPY ${APP} /${APP}
+
+
diff --git a/k8s/examples/vineyard-csidriver/Makefile b/k8s/examples/vineyard-csidriver/Makefile
new file mode 100644
index 00000000..b82b6ecc
--- /dev/null
+++ b/k8s/examples/vineyard-csidriver/Makefile
@@ -0,0 +1,22 @@
+docker-build:
+	docker build prepare-data/ -f Dockerfile \
+		--build-arg APP=prepare-data.py \
+		-t prepare-data
+
+	docker build preprocess/ -f Dockerfile \
+		--build-arg APP=preprocess.py \
+		-t preprocess-data
+
+	docker build train/ -f Dockerfile \
+		--build-arg APP=train.py \
+		-t train-data
+	
+	docker build test/ -f Dockerfile \
+		--build-arg APP=test.py \
+		-t test-data
+
+load-images:
+	kind load docker-image prepare-data
+	kind load docker-image preprocess-data
+	kind load docker-image train-data
+	kind load docker-image test-data
\ No newline at end of file
diff --git a/k8s/examples/vineyard-csidriver/pipeline-with-vineyard.py b/k8s/examples/vineyard-csidriver/pipeline-with-vineyard.py
new file mode 100644
index 00000000..6d41d30c
--- /dev/null
+++ b/k8s/examples/vineyard-csidriver/pipeline-with-vineyard.py
@@ -0,0 +1,82 @@
+from kfp import dsl
+
+def PreProcess():
+    #############################################################
+    # user need to add the volume Op for vineyard object manully
+    v1op = dsl.VolumeOp(name="vineyard-object1",
+                    resource_name="vineyard-object1-pvc", size="1Mi",
+                    storage_class="vineyard-system.vineyardd-sample.csi",
+                    modes=dsl.VOLUME_MODE_RWM,
+                    set_owner_reference=True)
+
+    v2op = dsl.VolumeOp(name="vineyard-object2",
+                resource_name="vineyard-object2-pvc", size="1Mi",
+                storage_class="vineyard-system.vineyardd-sample.csi",
+                modes=dsl.VOLUME_MODE_RWM,
+                set_owner_reference=True)
+
+    v3op = dsl.VolumeOp(name="vineyard-object3",
+            resource_name="vineyard-object3-pvc", size="1Mi",
+            storage_class="vineyard-system.vineyardd-sample.csi",
+            modes=dsl.VOLUME_MODE_RWM,
+            set_owner_reference=True)
+
+    v4op = dsl.VolumeOp(name="vineyard-object4",
+                    resource_name="vineyard-object4-pvc", size="1Mi",
+                    storage_class="vineyard-system.vineyardd-sample.csi",
+                    modes=dsl.VOLUME_MODE_RWM,
+                    set_owner_reference=True)
+    #############################################################
+
+    return dsl.ContainerOp(
+        name='Preprocess Data',
+        image = 'preprocess-data',
+        container_kwargs={'image_pull_policy':"IfNotPresent"},
+        pvolumes={
+            "/data": dsl.PipelineVolume(pvc="benchmark-data"),
+            '/data/x_train.pkl': v1op.volume,
+            '/data/y_train.pkl': v2op.volume,
+            '/data/x_test.pkl': v3op.volume,
+            '/data/y_test.pkl': v4op.volume
+        },
+        command = ['python3', 'preprocess.py']
+    )
+
+def Train(comp1):
+    return dsl.ContainerOp(
+        name='Train Data',
+        image='train-data',
+        container_kwargs={'image_pull_policy':"IfNotPresent"},
+        pvolumes={
+            "/data": comp1.pvolumes['/data'],
+            '/data/x_train.pkl': comp1.pvolumes['/data/x_train.pkl'],
+            '/data/y_train.pkl': comp1.pvolumes['/data/y_train.pkl']
+        },
+        command = ['python3', 'train.py'],
+    )
+
+def Test(comp1, comp2):
+    return dsl.ContainerOp(
+        name='Test Data',
+        image='test-data',
+        container_kwargs={'image_pull_policy':"IfNotPresent"},
+        pvolumes={
+            "/data": comp2.pvolumes['/data'],
+            '/data/x_test.pkl': comp1.pvolumes['/data/x_test.pkl'],
+            '/data/y_test.pkl': comp1.pvolumes['/data/y_test.pkl']
+        },
+        command = ['python3', 'test.py']
+    )
+
+@dsl.pipeline(
+   name='Machine learning Pipeline',
+   description='An example pipeline that trains and logs a regression model.'
+)
+def pipeline():
+    comp1 = PreProcess()
+    comp2 = Train(comp1)
+    comp3 = Test(comp1, comp2)
+
+if __name__ == '__main__':
+    from kfp import compiler
+    compiler.Compiler().compile(pipeline, __file__[:-3]+ '.yaml')
diff --git a/k8s/examples/vineyard-csidriver/pipeline-with-vineyard.yaml b/k8s/examples/vineyard-csidriver/pipeline-with-vineyard.yaml
new file mode 100644
index 00000000..1b943fc6
--- /dev/null
+++ b/k8s/examples/vineyard-csidriver/pipeline-with-vineyard.yaml
@@ -0,0 +1,264 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: machine-learning-pipeline-
+  annotations: {pipelines.kubeflow.org/kfp_sdk_version: 1.8.0, pipelines.kubeflow.org/pipeline_compilation_time: '2023-09-13T13:58:28.716735',
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "An example pipeline that
+      trains and logs a regression model.", "name": "Machine learning Pipeline"}'}
+  labels: {pipelines.kubeflow.org/kfp_sdk_version: 1.8.0}
+spec:
+  entrypoint: machine-learning-pipeline
+  templates:
+  - name: machine-learning-pipeline
+    dag:
+      tasks:
+      - name: preprocess-data
+        template: preprocess-data
+        dependencies: [vineyard-object1, vineyard-object2, vineyard-object3, vineyard-object4]
+        arguments:
+          parameters:
+          - {name: vineyard-object1-name, value: '{{tasks.vineyard-object1.outputs.parameters.vineyard-object1-name}}'}
+          - {name: vineyard-object2-name, value: '{{tasks.vineyard-object2.outputs.parameters.vineyard-object2-name}}'}
+          - {name: vineyard-object3-name, value: '{{tasks.vineyard-object3.outputs.parameters.vineyard-object3-name}}'}
+          - {name: vineyard-object4-name, value: '{{tasks.vineyard-object4.outputs.parameters.vineyard-object4-name}}'}
+      - name: test-data
+        template: test-data
+        dependencies: [preprocess-data, train-data, vineyard-object3, vineyard-object4]
+        arguments:
+          parameters:
+          - {name: vineyard-object3-name, value: '{{tasks.vineyard-object3.outputs.parameters.vineyard-object3-name}}'}
+          - {name: vineyard-object4-name, value: '{{tasks.vineyard-object4.outputs.parameters.vineyard-object4-name}}'}
+      - name: train-data
+        template: train-data
+        dependencies: [preprocess-data, vineyard-object1, vineyard-object2]
+        arguments:
+          parameters:
+          - {name: vineyard-object1-name, value: '{{tasks.vineyard-object1.outputs.parameters.vineyard-object1-name}}'}
+          - {name: vineyard-object2-name, value: '{{tasks.vineyard-object2.outputs.parameters.vineyard-object2-name}}'}
+      - {name: vineyard-object1, template: vineyard-object1}
+      - {name: vineyard-object2, template: vineyard-object2}
+      - {name: vineyard-object3, template: vineyard-object3}
+      - {name: vineyard-object4, template: vineyard-object4}
+  - name: preprocess-data
+    container:
+      command: [python3, preprocess.py]
+      image: preprocess-data
+      imagePullPolicy: IfNotPresent
+      securityContext:
+        privileged: true
+      env:
+      - name: WITH_VINEYARD
+        value: "true"
+      - name: DATA_MULTIPLIER
+        value: "{{workflow.parameters.data_multiplier}}"
+      volumeMounts:
+      - {mountPath: /data, name: pvolume-d9c6725a1237b14c08a2567cb12c489bec539873deeddba7d87f5b4}
+      - {mountPath: /data/x_train.pkl, name: vineyard-object1}
+      - {mountPath: /data/y_train.pkl, name: vineyard-object2}
+      - {mountPath: /data/x_test.pkl, name: vineyard-object3}
+      - {mountPath: /data/y_test.pkl, name: vineyard-object4}
+    inputs:
+      parameters:
+      - {name: vineyard-object1-name}
+      - {name: vineyard-object2-name}
+      - {name: vineyard-object3-name}
+      - {name: vineyard-object4-name}
+    metadata:
+      labels:
+        pipelines.kubeflow.org/kfp_sdk_version: 1.8.0
+        pipelines.kubeflow.org/pipeline-sdk-type: kfp
+        pipelines.kubeflow.org/enable_caching: "true"
+    volumes:
+    - name: pvolume-d9c6725a1237b14c08a2567cb12c489bec539873deeddba7d87f5b4
+      persistentVolumeClaim: {claimName: benchmark-data}
+    - name: vineyard-object1
+      persistentVolumeClaim: {claimName: '{{inputs.parameters.vineyard-object1-name}}'}
+    - name: vineyard-object2
+      persistentVolumeClaim: {claimName: '{{inputs.parameters.vineyard-object2-name}}'}
+    - name: vineyard-object3
+      persistentVolumeClaim: {claimName: '{{inputs.parameters.vineyard-object3-name}}'}
+    - name: vineyard-object4
+      persistentVolumeClaim: {claimName: '{{inputs.parameters.vineyard-object4-name}}'}
+  - name: test-data
+    container:
+      command: [python3, test.py]
+      image: test-data
+      imagePullPolicy: IfNotPresent
+      securityContext:
+        privileged: true
+      env:
+      - name: WITH_VINEYARD
+        value: "true"
+      - name: DATA_MULTIPLIER
+        value: "{{workflow.parameters.data_multiplier}}"
+      volumeMounts:
+      - {mountPath: /data, name: pvolume-d9c6725a1237b14c08a2567cb12c489bec539873deeddba7d87f5b4}
+      - {mountPath: /data/x_test.pkl, name: vineyard-object3}
+      - {mountPath: /data/y_test.pkl, name: vineyard-object4}
+    inputs:
+      parameters:
+      - {name: vineyard-object3-name}
+      - {name: vineyard-object4-name}
+    metadata:
+      labels:
+        pipelines.kubeflow.org/kfp_sdk_version: 1.8.0
+        pipelines.kubeflow.org/pipeline-sdk-type: kfp
+        pipelines.kubeflow.org/enable_caching: "true"
+    volumes:
+    - name: pvolume-d9c6725a1237b14c08a2567cb12c489bec539873deeddba7d87f5b4
+      persistentVolumeClaim: {claimName: benchmark-data}
+    - name: vineyard-object3
+      persistentVolumeClaim: {claimName: '{{inputs.parameters.vineyard-object3-name}}'}
+    - name: vineyard-object4
+      persistentVolumeClaim: {claimName: '{{inputs.parameters.vineyard-object4-name}}'}
+  - name: train-data
+    container:
+      command: [python3, train.py]
+      image: train-data
+      imagePullPolicy: IfNotPresent
+      securityContext:
+        privileged: true
+      env:
+      - name: WITH_VINEYARD
+        value: "true"
+      - name: DATA_MULTIPLIER
+        value: "{{workflow.parameters.data_multiplier}}"
+      volumeMounts:
+      - {mountPath: /data, name: pvolume-d9c6725a1237b14c08a2567cb12c489bec539873deeddba7d87f5b4}
+      - {mountPath: /data/x_train.pkl, name: vineyard-object1}
+      - {mountPath: /data/y_train.pkl, name: vineyard-object2}
+    inputs:
+      parameters:
+      - {name: vineyard-object1-name}
+      - {name: vineyard-object2-name}
+    metadata:
+      labels:
+        pipelines.kubeflow.org/kfp_sdk_version: 1.8.0
+        pipelines.kubeflow.org/pipeline-sdk-type: kfp
+        pipelines.kubeflow.org/enable_caching: "true"
+    volumes:
+    - name: pvolume-d9c6725a1237b14c08a2567cb12c489bec539873deeddba7d87f5b4
+      persistentVolumeClaim: {claimName: benchmark-data}
+    - name: vineyard-object1
+      persistentVolumeClaim: {claimName: '{{inputs.parameters.vineyard-object1-name}}'}
+    - name: vineyard-object2
+      persistentVolumeClaim: {claimName: '{{inputs.parameters.vineyard-object2-name}}'}
+  - name: vineyard-object1
+    resource:
+      action: create
+      setOwnerReference: true
+      manifest: |
+        apiVersion: v1
+        kind: PersistentVolumeClaim
+        metadata:
+          name: '{{workflow.name}}-vineyard-object1-pvc'
+        spec:
+          accessModes:
+          - ReadWriteMany
+          resources:
+            requests:
+              storage: 1Mi
+          storageClassName: vineyard-system.vineyardd-sample.csi
+    outputs:
+      parameters:
+      - name: vineyard-object1-manifest
+        valueFrom: {jsonPath: '{}'}
+      - name: vineyard-object1-name
+        valueFrom: {jsonPath: '{.metadata.name}'}
+      - name: vineyard-object1-size
+        valueFrom: {jsonPath: '{.status.capacity.storage}'}
+    metadata:
+      labels:
+        pipelines.kubeflow.org/kfp_sdk_version: 1.8.0
+        pipelines.kubeflow.org/pipeline-sdk-type: kfp
+        pipelines.kubeflow.org/enable_caching: "true"
+  - name: vineyard-object2
+    resource:
+      action: create
+      setOwnerReference: true
+      manifest: |
+        apiVersion: v1
+        kind: PersistentVolumeClaim
+        metadata:
+          name: '{{workflow.name}}-vineyard-object2-pvc'
+        spec:
+          accessModes:
+          - ReadWriteMany
+          resources:
+            requests:
+              storage: 1Mi
+          storageClassName: vineyard-system.vineyardd-sample.csi
+    outputs:
+      parameters:
+      - name: vineyard-object2-manifest
+        valueFrom: {jsonPath: '{}'}
+      - name: vineyard-object2-name
+        valueFrom: {jsonPath: '{.metadata.name}'}
+      - name: vineyard-object2-size
+        valueFrom: {jsonPath: '{.status.capacity.storage}'}
+    metadata:
+      labels:
+        pipelines.kubeflow.org/kfp_sdk_version: 1.8.0
+        pipelines.kubeflow.org/pipeline-sdk-type: kfp
+        pipelines.kubeflow.org/enable_caching: "true"
+  - name: vineyard-object3
+    resource:
+      action: create
+      setOwnerReference: true
+      manifest: |
+        apiVersion: v1
+        kind: PersistentVolumeClaim
+        metadata:
+          name: '{{workflow.name}}-vineyard-object3-pvc'
+        spec:
+          accessModes:
+          - ReadWriteMany
+          resources:
+            requests:
+              storage: 1Mi
+          storageClassName: vineyard-system.vineyardd-sample.csi
+    outputs:
+      parameters:
+      - name: vineyard-object3-manifest
+        valueFrom: {jsonPath: '{}'}
+      - name: vineyard-object3-name
+        valueFrom: {jsonPath: '{.metadata.name}'}
+      - name: vineyard-object3-size
+        valueFrom: {jsonPath: '{.status.capacity.storage}'}
+    metadata:
+      labels:
+        pipelines.kubeflow.org/kfp_sdk_version: 1.8.0
+        pipelines.kubeflow.org/pipeline-sdk-type: kfp
+        pipelines.kubeflow.org/enable_caching: "true"
+  - name: vineyard-object4
+    resource:
+      action: create
+      setOwnerReference: true
+      manifest: |
+        apiVersion: v1
+        kind: PersistentVolumeClaim
+        metadata:
+          name: '{{workflow.name}}-vineyard-object4-pvc'
+        spec:
+          accessModes:
+          - ReadWriteMany
+          resources:
+            requests:
+              storage: 1Mi
+          storageClassName: vineyard-system.vineyardd-sample.csi
+    outputs:
+      parameters:
+      - name: vineyard-object4-manifest
+        valueFrom: {jsonPath: '{}'}
+      - name: vineyard-object4-name
+        valueFrom: {jsonPath: '{.metadata.name}'}
+      - name: vineyard-object4-size
+        valueFrom: {jsonPath: '{.status.capacity.storage}'}
+    metadata:
+      labels:
+        pipelines.kubeflow.org/kfp_sdk_version: 1.8.0
+        pipelines.kubeflow.org/pipeline-sdk-type: kfp
+        pipelines.kubeflow.org/enable_caching: "true"
+  arguments:
+    parameters: []
+  serviceAccountName: pipeline-runner
diff --git a/k8s/examples/vineyard-csidriver/pipeline.py b/k8s/examples/vineyard-csidriver/pipeline.py
new file mode 100644
index 00000000..47c748e0
--- /dev/null
+++ b/k8s/examples/vineyard-csidriver/pipeline.py
@@ -0,0 +1,45 @@
+from kfp import dsl
+
+def PreProcess():
+    return dsl.ContainerOp(
+        name='Preprocess Data',
+        image = 'preprocess-data',
+        container_kwargs={'image_pull_policy':"IfNotPresent"},
+        command = ['python3', 'preprocess.py'],
+
+        # add the existing volume to the pipeline
+        pvolumes={"/data": dsl.PipelineVolume(pvc="benchmark-data")},
+    )
+
+def Train(comp1):
+    return dsl.ContainerOp(
+        name='Train Data',
+        image='train-data',
+        container_kwargs={'image_pull_policy':"IfNotPresent"},
+        command = ['python3', 'train.py'],
+        
+        pvolumes={"/data": comp1.pvolumes['/data']},
+    )
+
+def Test(comp2):
+    return dsl.ContainerOp(
+        name='Test Data',
+        image='test-data',
+        container_kwargs={'image_pull_policy':"IfNotPresent"},
+        command = ['python3', 'test.py'],
+
+        pvolumes={"/data": comp2.pvolumes['/data']},
+    )
+
+@dsl.pipeline(
+   name='Machine Learning Pipeline',
+   description='An example pipeline that trains and logs a regression model.'
+)
+def pipeline():
+    comp1 = PreProcess()
+    comp2 = Train(comp1)
+    comp3 = Test(comp2)
+
+if __name__ == '__main__':
+    from kfp import compiler
+    compiler.Compiler().compile(pipeline, __file__[:-3]+ '.yaml')
diff --git a/k8s/examples/vineyard-csidriver/pipeline.yaml b/k8s/examples/vineyard-csidriver/pipeline.yaml
new file mode 100644
index 00000000..cf45909b
--- /dev/null
+++ b/k8s/examples/vineyard-csidriver/pipeline.yaml
@@ -0,0 +1,84 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: machine-learning-pipeline-
+  annotations: {pipelines.kubeflow.org/kfp_sdk_version: 1.8.0, pipelines.kubeflow.org/pipeline_compilation_time: '2023-09-13T13:57:48.817920',
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "An example pipeline that
+      trains and logs a regression model.", "name": "Machine Learning Pipeline"}'}
+  labels: {pipelines.kubeflow.org/kfp_sdk_version: 1.8.0}
+spec:
+  entrypoint: machine-learning-pipeline
+  templates:
+  - name: machine-learning-pipeline
+    dag:
+      tasks:
+      - {name: preprocess-data, template: preprocess-data}
+      - name: test-data
+        template: test-data
+        dependencies: [train-data]
+      - name: train-data
+        template: train-data
+        dependencies: [preprocess-data]
+  - name: preprocess-data
+    container:
+      command: [python3, preprocess.py]
+      image: preprocess-data
+      imagePullPolicy: IfNotPresent
+      securityContext:
+        privileged: true
+      env:
+      - name: DATA_MULTIPLIER
+        value: "{{workflow.parameters.data_multiplier}}"
+      volumeMounts:
+      - {mountPath: /data, name: pvolume-d9c6725a1237b14c08a2567cb12c489bec539873deeddba7d87f5b4}
+    metadata:
+      labels:
+        pipelines.kubeflow.org/kfp_sdk_version: 1.8.0
+        pipelines.kubeflow.org/pipeline-sdk-type: kfp
+        pipelines.kubeflow.org/enable_caching: "true"
+    volumes:
+    - name: pvolume-d9c6725a1237b14c08a2567cb12c489bec539873deeddba7d87f5b4
+      persistentVolumeClaim: {claimName: benchmark-data}
+  - name: test-data
+    container:
+      command: [python3, test.py]
+      image: test-data
+      imagePullPolicy: IfNotPresent
+      securityContext:
+        privileged: true
+      env:
+      - name: DATA_MULTIPLIER
+        value: "{{workflow.parameters.data_multiplier}}"
+      volumeMounts:
+      - {mountPath: /data, name: pvolume-d9c6725a1237b14c08a2567cb12c489bec539873deeddba7d87f5b4}
+    metadata:
+      labels:
+        pipelines.kubeflow.org/kfp_sdk_version: 1.8.0
+        pipelines.kubeflow.org/pipeline-sdk-type: kfp
+        pipelines.kubeflow.org/enable_caching: "true"
+    volumes:
+    - name: pvolume-d9c6725a1237b14c08a2567cb12c489bec539873deeddba7d87f5b4
+      persistentVolumeClaim: {claimName: benchmark-data}
+  - name: train-data
+    container:
+      command: [python3, train.py]
+      image: train-data
+      imagePullPolicy: IfNotPresent
+      securityContext:
+        privileged: true
+      env:
+      - name: DATA_MULTIPLIER
+        value: "{{workflow.parameters.data_multiplier}}"
+      volumeMounts:
+      - {mountPath: /data, name: pvolume-d9c6725a1237b14c08a2567cb12c489bec539873deeddba7d87f5b4}
+    metadata:
+      labels:
+        pipelines.kubeflow.org/kfp_sdk_version: 1.8.0
+        pipelines.kubeflow.org/pipeline-sdk-type: kfp
+        pipelines.kubeflow.org/enable_caching: "true"
+    volumes:
+    - name: pvolume-d9c6725a1237b14c08a2567cb12c489bec539873deeddba7d87f5b4
+      persistentVolumeClaim: {claimName: benchmark-data}
+  arguments:
+    parameters: []
+  serviceAccountName: pipeline-runner
diff --git a/k8s/examples/vineyard-csidriver/prepare-data.yaml b/k8s/examples/vineyard-csidriver/prepare-data.yaml
new file mode 100644
index 00000000..329d8a09
--- /dev/null
+++ b/k8s/examples/vineyard-csidriver/prepare-data.yaml
@@ -0,0 +1,54 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prepare-data
+spec:
+  selector:
+    matchLabels:
+      app: prepare-data
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: prepare-data
+    spec:
+      containers:
+      - name: prepare-data
+        image: prepare-data
+        imagePullPolicy: IfNotPresent
+        command: ["python3", "/prepare-data.py"]
+        volumeMounts:
+        - mountPath: /data
+          name: data
+      volumes:
+      - name: data
+        persistentVolumeClaim:
+          claimName: benchmark-data
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: benchmark-data
+spec:
+  storageClassName: manual
+  accessModes:
+  - ReadWriteMany
+  resources:
+    requests:
+      storage: 30Gi
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: benchmark-data
+  labels:
+    type: local
+spec:
+  storageClassName: manual
+  capacity:
+    storage: 30Gi
+  accessModes:
+    - ReadWriteMany
+  hostPath:
+    # mount a nfs volume to the kind nodes
+    path: "/mnt/csi-benchmark"
\ No newline at end of file
diff --git a/k8s/examples/vineyard-csidriver/prepare-data/prepare-data.py b/k8s/examples/vineyard-csidriver/prepare-data/prepare-data.py
new file mode 100644
index 00000000..491e6ad5
--- /dev/null
+++ b/k8s/examples/vineyard-csidriver/prepare-data/prepare-data.py
@@ -0,0 +1,68 @@
+import time
+
+import numpy as np
+import pandas as pd
+
+
+def generate_random_dataframe(num_rows):
+    return pd.DataFrame({
+            'Id': np.random.randint(1, 100000, num_rows),
+            'MSSubClass': np.random.randint(20, 201, size=num_rows),
+            'LotFrontage': np.random.randint(50, 151, size=num_rows),
+            'LotArea': np.random.randint(5000, 20001, size=num_rows),
+            'OverallQual': np.random.randint(1, 11, size=num_rows),
+            'OverallCond': np.random.randint(1, 11, size=num_rows),
+            'YearBuilt': np.random.randint(1900, 2022, size=num_rows),
+            'YearRemodAdd': np.random.randint(1900, 2022, size=num_rows),
+            'MasVnrArea': np.random.randint(0, 1001, size=num_rows),
+            'BsmtFinSF1': np.random.randint(0, 2001, size=num_rows),
+            'BsmtFinSF2': np.random.randint(0, 1001, size=num_rows),
+            'BsmtUnfSF': np.random.randint(0, 2001, size=num_rows),
+            'TotalBsmtSF': np.random.randint(0, 3001, size=num_rows),
+            '1stFlrSF': np.random.randint(500, 4001, size=num_rows),
+            '2ndFlrSF': np.random.randint(0, 2001, size=num_rows),
+            'LowQualFinSF': np.random.randint(0, 201, size=num_rows),
+            'GrLivArea': np.random.randint(600, 5001, size=num_rows),
+            'BsmtFullBath': np.random.randint(0, 4, size=num_rows),
+            'BsmtHalfBath': np.random.randint(0, 3, size=num_rows),
+            'FullBath': np.random.randint(0, 5, size=num_rows),
+            'HalfBath': np.random.randint(0, 3, size=num_rows),
+            'BedroomAbvGr': np.random.randint(0, 11, size=num_rows),
+            'KitchenAbvGr': np.random.randint(0, 4, size=num_rows),
+            'TotRmsAbvGrd': np.random.randint(0, 16, size=num_rows),
+            'Fireplaces': np.random.randint(0, 4, size=num_rows),
+            'GarageYrBlt': np.random.randint(1900, 2022, size=num_rows),
+            'GarageCars': np.random.randint(0, 5, num_rows),
+            'GarageArea': np.random.randint(0, 1001, num_rows),
+            'WoodDeckSF': np.random.randint(0, 501, num_rows),
+            'OpenPorchSF': np.random.randint(0, 301, num_rows),
+            'EnclosedPorch': np.random.randint(0, 201, num_rows),
+            '3SsnPorch': np.random.randint(0, 101, num_rows),
+            'ScreenPorch': np.random.randint(0, 201, num_rows),
+            'PoolArea': np.random.randint(0, 301, num_rows),
+            'MiscVal': np.random.randint(0, 5001, num_rows),
+            'MoSold': np.random.randint(1, 13, num_rows),
+            'YrSold': np.random.randint(2006, 2022, num_rows),
+            'SalePrice': np.random.randint(50000, 800001, num_rows),
+        })
+
+def prepare_data():
+    print('Start preparing data....', flush=True)
+    st = time.time()
+    for multiplier in 3000, 4000, 5000:
+        df = generate_random_dataframe(10000*(multiplier))
+        df.to_pickle('/data/df_{}.pkl'.format(multiplier))
+        del df
+    ed = time.time()
+    print('##################################', flush=True)
+    print('dataframe to_pickle time: ', ed - st, flush=True)
+
+
+if __name__ == '__main__':
+    st = time.time()
+    print('Preparing data....', flush=True)
+    prepare_data()
+    ed = time.time()
+    print('##################################')
+    print('preparing data time: ', ed - st, flush=True)
+    time.sleep(10000000)
diff --git a/k8s/examples/vineyard-csidriver/preprocess/preprocess.py b/k8s/examples/vineyard-csidriver/preprocess/preprocess.py
new file mode 100644
index 00000000..64237f0f
--- /dev/null
+++ b/k8s/examples/vineyard-csidriver/preprocess/preprocess.py
@@ -0,0 +1,85 @@
+import os
+import time
+
+#from sklearn.compose import ColumnTransformer
+from sklearn.model_selection import train_test_split
+#from sklearn.preprocessing import OneHotEncoder
+
+import pandas as pd
+import vineyard
+
+
+def preprocess_data():
+    os.system('sync; echo 3 > /proc/sys/vm/drop_caches')
+    data_multiplier = os.environ.get('DATA_MULTIPLIER', 1)
+    st = time.time()
+    df = pd.read_pickle('/data/df_{0}.pkl'.format(data_multiplier))
+
+    ed = time.time()
+    print('##################################')
+    print('read dataframe pickle time: ', ed - st)
+
+    df = df.drop(df[(df['GrLivArea']>4800)].index)
+
+    """ The following part will need large memory usage, disable for benchmark
+    del df
+
+    # Define the categorical feature columns
+    categorical_features = df_preocessed.select_dtypes(include='object').columns
+
+    # Create the column transformer for one-hot encoding
+    preprocessor = ColumnTransformer(
+        transformers=[('encoder', OneHotEncoder(sparse=False, handle_unknown='ignore'), categorical_features)],
+        remainder='passthrough'
+    )
+
+    # Preprocess the features using the column transformer
+    one_hot_df = preprocessor.fit_transform(df_preocessed)
+
+    # Get the column names for the encoded features
+    encoded_feature_names = preprocessor.named_transformers_['encoder'].get_feature_names_out(categorical_features)
+
+    columns = list(encoded_feature_names) + list(df_preocessed.select_dtypes(exclude='object').columns)
+
+    del df_preocessed
+
+    # Concatenate the encoded features with the original numerical features
+    df = pd.DataFrame(one_hot_df, columns=columns)
+
+    del one_hot_df
+    """
+
+    X = df.drop('SalePrice', axis=1)  # Features
+    y = df['SalePrice']  # Target variable
+
+    del df
+
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1)
+
+    del X, y
+
+    st = time.time()
+    with_vineyard = os.environ.get('WITH_VINEYARD', False)
+    if with_vineyard:
+        vineyard.csi.write(X_train, "/data/x_train.pkl")
+        vineyard.csi.write(X_test, "/data/x_test.pkl")
+        vineyard.csi.write(y_train, "/data/y_train.pkl")
+        vineyard.csi.write(y_test, "/data/y_test.pkl")
+    else:
+        X_train.to_pickle('/data/x_train.pkl')
+        X_test.to_pickle('/data/x_test.pkl')
+        y_train.to_pickle('/data/y_train.pkl')
+        y_test.to_pickle('/data/y_test.pkl')
+
+    ed = time.time()
+    print('##################################')
+    print('write training and testing data time: ', ed - st)
+
+
+if __name__ == '__main__':
+    st = time.time()
+    print('Preprocessing data...')
+    preprocess_data()
+    ed = time.time()
+    print('##################################')
+    print('Preprocessing data time: ', ed - st)
diff --git a/k8s/examples/vineyard-csidriver/rbac.yaml b/k8s/examples/vineyard-csidriver/rbac.yaml
new file mode 100644
index 00000000..3d791138
--- /dev/null
+++ b/k8s/examples/vineyard-csidriver/rbac.yaml
@@ -0,0 +1,30 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: pipeline-runner
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: pipeline-runner-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "create", "update", "list", "delete"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "patch", "create", "update", "list", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: pipeline-runner-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: pipeline-runner-role
+subjects:
+  - kind: ServiceAccount
+    name: pipeline-runner
+    namespace: default
\ No newline at end of file
diff --git a/k8s/examples/vineyard-csidriver/test/test.py b/k8s/examples/vineyard-csidriver/test/test.py
new file mode 100644
index 00000000..97d5d0da
--- /dev/null
+++ b/k8s/examples/vineyard-csidriver/test/test.py
@@ -0,0 +1,42 @@
+import os
+import time
+
+from sklearn.metrics import mean_squared_error
+
+import joblib
+import pandas as pd
+import vineyard
+
+def test_model():
+    os.system('sync; echo 3 > /proc/sys/vm/drop_caches')
+    with_vineyard = os.environ.get('WITH_VINEYARD', False)
+    st = time.time()
+    if with_vineyard:
+        x_test_data = vineyard.csi.read("/data/x_test.pkl")
+        y_test_data = vineyard.csi.read("/data/y_test.pkl")
+    else:
+        x_test_data = pd.read_pickle("/data/x_test.pkl")
+        y_test_data = pd.read_pickle("/data/y_test.pkl")
+        #delete the x_test.pkl and y_test.pkl
+        os.remove("/data/x_test.pkl")
+        os.remove("/data/y_test.pkl")
+    ed = time.time()
+    print('##################################')
+    print('read x_test and y_test execution time: ', ed - st)
+
+    model = joblib.load("/data/model.pkl")
+    y_pred = model.predict(x_test_data)
+
+    err = mean_squared_error(y_test_data, y_pred)
+
+    with open('/data/output.txt', 'a') as f:
+        f.write(str(err))
+
+
+if __name__ == '__main__':
+    st = time.time()
+    print('Testing model...')
+    test_model()
+    ed = time.time()
+    print('##################################')
+    print('Testing model data time: ', ed - st)
diff --git a/k8s/examples/vineyard-csidriver/train/train.py b/k8s/examples/vineyard-csidriver/train/train.py
new file mode 100644
index 00000000..93a71c77
--- /dev/null
+++ b/k8s/examples/vineyard-csidriver/train/train.py
@@ -0,0 +1,40 @@
+import os
+import time
+
+from sklearn.linear_model import LinearRegression
+
+import joblib
+import pandas as pd
+import vineyard
+
+
+def train_model():
+    os.system('echo 3 > /proc/sys/vm/drop_caches')
+    st = time.time()
+    with_vineyard = os.environ.get('WITH_VINEYARD', False)
+    if with_vineyard:
+        x_train_data = vineyard.csi.read("/data/x_train.pkl")
+        y_train_data = vineyard.csi.read("/data/y_train.pkl")
+    else:
+        x_train_data = pd.read_pickle("/data/x_train.pkl")
+        y_train_data = pd.read_pickle("/data/y_train.pkl")
+        # delete the x_train.pkl and y_train.pkl
+        os.remove("/data/x_train.pkl")
+        os.remove("/data/y_train.pkl")
+    ed = time.time()
+    print('##################################')
+    print('read x_train and y_train data time: ', ed - st)
+
+    model = LinearRegression()
+    model.fit(x_train_data, y_train_data)
+
+    joblib.dump(model, '/data/model.pkl')
+
+
+if __name__ == '__main__':
+    st = time.time()
+    print('Training model...')
+    train_model()
+    ed = time.time()
+    print('##################################')
+    print('Training model data time: ', ed - st)
diff --git a/k8s/go.mod b/k8s/go.mod
index 01935d1d..6c3db166 100644
--- a/k8s/go.mod
+++ b/k8s/go.mod
@@ -7,8 +7,12 @@ require (
 	github.com/argoproj/argo-workflows/v3 v3.3.10
 	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/cert-manager/cert-manager v1.8.0
+	github.com/container-storage-interface/spec v1.6.0
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-logr/logr v1.2.3
+	github.com/google/uuid v1.3.0
+	github.com/kubernetes-csi/csi-lib-utils v0.0.0-00010101000000-000000000000
+	github.com/kubernetes-csi/csi-test/v4 v4.4.0
 	github.com/olekukonko/tablewriter v0.0.4
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.4.0
@@ -17,6 +21,7 @@ require (
 	github.com/v6d-io/v6d/go/vineyard v0.0.0-00010101000000-000000000000
 	go.uber.org/multierr v1.9.0
 	go.uber.org/zap v1.24.0
+	google.golang.org/grpc v1.53.0
 	k8s.io/api v0.25.10
 	k8s.io/apiextensions-apiserver v0.25.10
 	k8s.io/apimachinery v0.25.10
@@ -24,6 +29,7 @@ require (
 	k8s.io/component-base v0.25.10
 	k8s.io/component-helpers v0.25.10
 	k8s.io/kubernetes v1.25.10
+	k8s.io/mount-utils v0.25.10
 	sigs.k8s.io/controller-runtime v0.13.0
 	sigs.k8s.io/kustomize/kustomize/v4 v4.5.7
 	sigs.k8s.io/kustomize/kyaml v0.13.9
@@ -63,7 +69,6 @@ require (
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/google/uuid v1.3.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
 	github.com/huandu/xstrings v1.3.2 // indirect
@@ -122,7 +127,6 @@ require (
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
-	google.golang.org/grpc v1.53.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
@@ -134,7 +138,6 @@ require (
 	k8s.io/klog/v2 v2.70.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
 	k8s.io/kube-scheduler v0.25.10 // indirect
-	k8s.io/mount-utils v0.25.10 // indirect
 	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.37 // indirect
 	sigs.k8s.io/gateway-api v0.4.1 // indirect
@@ -149,8 +152,9 @@ replace (
 	//github.com/emicklei/go-restful => github.com/emicklei/go-restful v2.9.5+incompatible
 	//github.com/emicklei/go-restful/v3 => github.com/emicklei/go-restful v2.9.5+incompatible
 	github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.4.2
-	github.com/v6d-io/v6d/go/vineyard => ../go/vineyard
 	// these are needed since k8s.io/kubernetes cites v0.0.0 for these in its go.mod
+	github.com/kubernetes-csi/csi-lib-utils => github.com/kubernetes-csi/csi-lib-utils v0.11.0
+	github.com/v6d-io/v6d/go/vineyard => ../go/vineyard
 	k8s.io/api => k8s.io/api v0.25.10
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.25.10
 	k8s.io/apimachinery => k8s.io/apimachinery v0.25.10
diff --git a/k8s/go.sum b/k8s/go.sum
index 2552f00c..d1a57186 100644
--- a/k8s/go.sum
+++ b/k8s/go.sum
@@ -135,6 +135,9 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH
 github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
+github.com/container-storage-interface/spec v1.5.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s=
+github.com/container-storage-interface/spec v1.6.0 h1:vwN9uCciKygX/a0toYryoYD5+qI9ZFeAMuhEEKO+JBA=
+github.com/container-storage-interface/spec v1.6.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
@@ -411,6 +414,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kubernetes-csi/csi-lib-utils v0.11.0 h1:FHWOBtAZBA/hVk7v/qaXgG9Sxv0/n06DebPFuDwumqg=
+github.com/kubernetes-csi/csi-lib-utils v0.11.0/go.mod h1:BmGZZB16L18+9+Lgg9YWwBKfNEHIDdgGfAyuW6p2NV0=
+github.com/kubernetes-csi/csi-test/v4 v4.4.0 h1:r0mnAwDURI24Vw3a/LyA/ga11yD5ZGuU7+REO35Na9s=
+github.com/kubernetes-csi/csi-test/v4 v4.4.0/go.mod h1:t1RzseMZJKy313nezI/d7TolbbiKpUZM3SXQvXxOX0w=
 github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
@@ -782,6 +789,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
 golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
@@ -1075,6 +1083,7 @@ google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201209185603-f92720507ed4/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
@@ -1181,8 +1190,8 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
+gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
-gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -1215,7 +1224,9 @@ k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAE
 k8s.io/klog v0.2.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
+k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 k8s.io/klog/v2 v2.10.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
+k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ=
 k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA=
@@ -1254,4 +1265,4 @@ sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kF
 sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
 sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
-sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
\ No newline at end of file
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
diff --git a/k8s/pkg/csidriver/controller.go b/k8s/pkg/csidriver/controller.go
new file mode 100644
index 00000000..bf0b7441
--- /dev/null
+++ b/k8s/pkg/csidriver/controller.go
@@ -0,0 +1,269 @@
+/*
+* Copyright 2020-2023 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package csidriver
+
+import (
+	"context"
+	"sync"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"github.com/google/uuid"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
+	"github.com/v6d-io/v6d/k8s/pkg/log"
+)
+
+var (
+	controllerCaps = []csi.ControllerServiceCapability_RPC_Type{
+		csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
+		csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME,
+	}
+	VineyardSocketPrefix = "/var/run/vineyard-kubernetes"
+	VineyardSocket       = "vineyard.sock"
+
+	// VineyardNamespaceKey is the key for the vineyard name in the volume context
+	VineyardNamespaceKey = "k8s.v6d.io/vineyard/namespace"
+
+	// VineyardNameKey is the key for the vineyard name in the volume context
+	VineyardNameKey = "k8s.v6d.io/vineyard/name"
+)
+
+type VineyardCSI struct {
+	mutex  sync.Mutex
+	state  *VolumeStates
+	nodeID string
+}
+
+func NewVineyardCSI(stateFilePath string, nodeID string) *VineyardCSI {
+	volumeStates, err := NewVolumeStates(stateFilePath)
+	if err != nil {
+		log.Fatalf(err, "failed to create volume states")
+	}
+	return &VineyardCSI{
+		mutex:  sync.Mutex{},
+		state:  volumeStates,
+		nodeID: nodeID,
+	}
+}
+
+func (vc *VineyardCSI) ControllerGetCapabilities(ctx context.Context,
+	req *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) {
+	log.Infof("ControllerGetCapabilities: called with args %+v", *req)
+	caps := []*csi.ControllerServiceCapability{}
+	for _, cap := range controllerCaps {
+		c := &csi.ControllerServiceCapability{
+			Type: &csi.ControllerServiceCapability_Rpc{
+				Rpc: &csi.ControllerServiceCapability_RPC{
+					Type: cap,
+				},
+			},
+		}
+		caps = append(caps, c)
+	}
+	return &csi.ControllerGetCapabilitiesResponse{Capabilities: caps}, nil
+}
+
+func (vc *VineyardCSI) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) {
+	log.Infof("CreateVolume: called with args %+v", *req)
+	// check arguments
+	if req.Name == "" {
+		return nil, status.Error(codes.InvalidArgument, "CreateVolume Name must be provided")
+	}
+
+	if req.VolumeCapabilities == nil || len(req.VolumeCapabilities) == 0 {
+		return nil, status.Error(codes.InvalidArgument, "CreateVolume Volume capabilities must be provided")
+	}
+	vc.mutex.Lock()
+	defer vc.mutex.Unlock()
+
+	capacity := req.GetCapacityRange().GetRequiredBytes()
+	// check whether the volume exists
+	if exVol, err := vc.state.GetVolumeByName(req.Name); err == nil {
+		if exVol.VolSize != capacity {
+			return nil, status.Errorf(codes.AlreadyExists, "Volume with the same name: %s but with different size already exist", req.GetName())
+		}
+
+		// volume already exists, return the volume
+		return &csi.CreateVolumeResponse{
+			Volume: &csi.Volume{
+				VolumeId:      exVol.VolID,
+				VolumeContext: req.GetParameters(),
+			},
+		}, nil
+	}
+	// create the uuid for the volume
+	uuid, err := uuid.NewUUID()
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	volumeID := uuid.String()
+	log.Infof("create volumeID: %s", volumeID)
+
+	volumeContext := req.GetParameters()
+	if len(volumeContext) == 0 {
+		volumeContext = make(map[string]string)
+	}
+	volumeContext["volume_id"] = volumeID
+
+	// add volume to the state
+	volume := Volume{
+		VolID:   volumeID,
+		VolName: req.Name,
+		VolSize: capacity,
+	}
+	if err := vc.state.AddVolume(volume); err != nil {
+		return nil, err
+	}
+
+	return &csi.CreateVolumeResponse{
+		Volume: &csi.Volume{
+			VolumeId:      volumeID,
+			VolumeContext: volumeContext,
+		},
+	}, nil
+}
+
+func (vc *VineyardCSI) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) {
+	log.Infof("DeleteVolume: called with args: %+v", *req)
+	if req.VolumeId == "" {
+		return nil, status.Error(codes.InvalidArgument, "DeleteVolume Volume ID must be provided")
+	}
+
+	if err := vc.state.DeleteVolume(req.VolumeId); err != nil {
+		return nil, err
+	}
+	return &csi.DeleteVolumeResponse{}, nil
+}
+
+func (vc *VineyardCSI) ControllerGetVolume(ctx context.Context, req *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error) {
+	log.Infof("ControllerGetVolume: called with args: %+v", *req)
+	return &csi.ControllerGetVolumeResponse{}, nil
+}
+
+func (vc *VineyardCSI) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) {
+	log.Infof("ControllerPublishVolume: called with args %+v", *req)
+	if req.VolumeId == "" {
+		return nil, status.Error(codes.InvalidArgument, "ControllerPublishVolume Volume ID must be provided")
+	}
+
+	if req.NodeId == "" {
+		return nil, status.Error(codes.InvalidArgument, "ControllerPublishVolume Node ID must be provided")
+	}
+
+	if req.VolumeCapability == nil {
+		return nil, status.Error(codes.InvalidArgument, "ControllerPublishVolume Volume capability must be provided")
+	}
+
+	vc.mutex.Lock()
+	defer vc.mutex.Unlock()
+
+	vol, err := vc.state.GetVolumeByID(req.GetVolumeId())
+	if err != nil {
+		return nil, err
+	}
+
+	if vol.Attched {
+		if req.GetReadonly() != vol.ReadOnlyAttach {
+			return nil, status.Error(codes.AlreadyExists, "Volume published but has incompatible readonly flag")
+		}
+
+		return &csi.ControllerPublishVolumeResponse{
+			PublishContext: map[string]string{},
+		}, nil
+	}
+
+	vol.Attched = true
+	vol.ReadOnlyAttach = req.GetReadonly()
+
+	if err := vc.state.AddVolume(vol); err != nil {
+		return nil, err
+	}
+	return &csi.ControllerPublishVolumeResponse{}, nil
+}
+
+func (vc *VineyardCSI) ControllerUnpublishVolume(ctx context.Context,
+	req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) {
+	log.Infof("ControllerUnpublishVolume: called with args %+v", *req)
+	if req.VolumeId == "" {
+		return nil, status.Error(codes.InvalidArgument, "ControllerPublishVolume Volume ID must be provided")
+	}
+	return &csi.ControllerUnpublishVolumeResponse{}, nil
+}
+
+func (vc *VineyardCSI) ValidateVolumeCapabilities(ctx context.Context,
+	req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) {
+	if req.VolumeId == "" {
+		return nil, status.Error(codes.InvalidArgument, "ValidateVolumeCapabilities Volume ID must be provided")
+	}
+
+	if req.VolumeCapabilities == nil {
+		return nil, status.Error(codes.InvalidArgument, "ValidateVolumeCapabilities Volume Capabilities must be provided")
+	}
+	vc.mutex.Lock()
+	defer vc.mutex.Unlock()
+
+	if _, err := vc.state.GetVolumeByID(req.GetVolumeId()); err != nil {
+		return nil, err
+	}
+
+	for _, cap := range req.GetVolumeCapabilities() {
+		if cap.GetMount() == nil && cap.GetBlock() == nil {
+			return nil, status.Error(codes.InvalidArgument, "cannot have both mount and block access type be undefined")
+		}
+	}
+
+	return &csi.ValidateVolumeCapabilitiesResponse{
+		Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{
+			VolumeContext:      req.GetVolumeContext(),
+			VolumeCapabilities: req.GetVolumeCapabilities(),
+			Parameters:         req.GetParameters(),
+		},
+	}, nil
+
+}
+
+func (vc *VineyardCSI) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "")
+}
+
+func (vc *VineyardCSI) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "")
+}
+
+func (vc *VineyardCSI) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) {
+	if req.GetName() == "" {
+		return nil, status.Error(codes.InvalidArgument, "CreateSnapshot Name must be provided")
+	}
+
+	if req.GetSourceVolumeId() == "" {
+		return nil, status.Error(codes.InvalidArgument, "CreateSnapshot Source Volume ID must be provided")
+	}
+
+	return nil, status.Error(codes.Unimplemented, "")
+}
+
+func (vc *VineyardCSI) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "")
+}
+
+func (vc *VineyardCSI) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "")
+}
+
+func (vc *VineyardCSI) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "")
+}
diff --git a/k8s/pkg/csidriver/driver.go b/k8s/pkg/csidriver/driver.go
new file mode 100644
index 00000000..96da0696
--- /dev/null
+++ b/k8s/pkg/csidriver/driver.go
@@ -0,0 +1,115 @@
+/*
+* Copyright 2020-2023 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package csidriver
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"strings"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"github.com/kubernetes-csi/csi-lib-utils/protosanitizer"
+	"github.com/pkg/errors"
+	"google.golang.org/grpc"
+
+	"github.com/v6d-io/v6d/k8s/cmd/commands/flags"
+	"github.com/v6d-io/v6d/k8s/pkg/log"
+)
+
+type Driver struct {
+	nodeID   string
+	endpoint string
+}
+
+const (
+	version    = "0.1.0"
+	driverName = "csi.vineyard.v6d.io"
+)
+
+func NewDriver(nodeID, endpoint string) *Driver {
+	log.Infof("Driver: %v version: %v", driverName, version)
+
+	n := &Driver{
+		nodeID:   nodeID,
+		endpoint: endpoint,
+	}
+
+	return n
+}
+
+func (d *Driver) Run() {
+
+	vineyardCSI := NewVineyardCSI(flags.StateFilePath, d.nodeID)
+	identity := NewIdentityServer()
+
+	opts := []grpc.ServerOption{}
+	if flags.Verbose {
+		opts = append(opts, grpc.UnaryInterceptor(logGRPC))
+	}
+
+	srv := grpc.NewServer(opts...)
+
+	csi.RegisterControllerServer(srv, vineyardCSI)
+	csi.RegisterIdentityServer(srv, identity)
+	csi.RegisterNodeServer(srv, vineyardCSI)
+
+	proto, addr, err := ParseEndpoint(d.endpoint)
+	log.Infof("protocol: %s,addr: %s", proto, addr)
+	if err != nil {
+		log.Fatalf(err, "Invalid endpoint: %v", d.endpoint)
+	}
+
+	if proto == "unix" {
+		addr = "/" + addr
+		if err := os.Remove(addr); err != nil && !os.IsNotExist(err) {
+			log.Fatalf(err, "Failed to remove %s, error: %s", addr, err.Error())
+		}
+	}
+
+	listener, err := net.Listen(proto, addr)
+	if err != nil {
+		log.Fatalf(err, "Failed to listen")
+	}
+
+	if err := srv.Serve(listener); err != nil {
+		log.Fatalf(err, "Failed to serve")
+	}
+}
+
+func logGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
+	log.Infof("GRPC call: %s", info.FullMethod)
+	log.Infof("GRPC request: %s", protosanitizer.StripSecrets(req))
+
+	resp, err := handler(ctx, req)
+	if err != nil {
+		log.Errorf(err, "GRPC error: %s", err.Error())
+	} else {
+		log.Infof("GRPC response: %s", protosanitizer.StripSecrets(resp))
+	}
+	return resp, err
+}
+
+func ParseEndpoint(ep string) (string, string, error) {
+	if strings.HasPrefix(strings.ToLower(ep), "unix://") || strings.HasPrefix(strings.ToLower(ep), "tcp://") {
+		s := strings.SplitN(ep, "://", 2)
+		if s[1] != "" {
+			return s[0], s[1], nil
+		}
+	}
+	return "", "", errors.Wrap(fmt.Errorf("invalid endpoint: %s", ep), "parse endpoint")
+}
diff --git a/k8s/pkg/csidriver/identity.go b/k8s/pkg/csidriver/identity.go
new file mode 100644
index 00000000..34fa9ee1
--- /dev/null
+++ b/k8s/pkg/csidriver/identity.go
@@ -0,0 +1,70 @@
+/*
+* Copyright 2020-2023 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package csidriver
+
+import (
+	"context"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+
+	"github.com/v6d-io/v6d/k8s/pkg/log"
+)
+
+type IdentityServer struct {
+}
+
+func NewIdentityServer() *IdentityServer {
+	return &IdentityServer{}
+}
+
+func (ids *IdentityServer) GetPluginInfo(ctx context.Context, req *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) {
+	log.Infof("GetPluginInfo: called with args %+v", *req)
+
+	return &csi.GetPluginInfoResponse{
+		Name:          driverName,
+		VendorVersion: version,
+	}, nil
+}
+
+func (ids *IdentityServer) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) {
+	log.Infof("GetPluginCapabilities: called with args %+v", *req)
+
+	resp := &csi.GetPluginCapabilitiesResponse{
+		Capabilities: []*csi.PluginCapability{
+			{
+				Type: &csi.PluginCapability_Service_{
+					Service: &csi.PluginCapability_Service{
+						Type: csi.PluginCapability_Service_CONTROLLER_SERVICE,
+					},
+				},
+			},
+			{
+				Type: &csi.PluginCapability_Service_{
+					Service: &csi.PluginCapability_Service{
+						Type: csi.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS,
+					},
+				},
+			},
+		},
+	}
+
+	return resp, nil
+}
+
+func (ids *IdentityServer) Probe(ctx context.Context, req *csi.ProbeRequest) (*csi.ProbeResponse, error) {
+	log.Infof("Probe: called with args %+v", *req)
+	return &csi.ProbeResponse{}, nil
+}
diff --git a/k8s/pkg/csidriver/node.go b/k8s/pkg/csidriver/node.go
new file mode 100644
index 00000000..11966b5d
--- /dev/null
+++ b/k8s/pkg/csidriver/node.go
@@ -0,0 +1,183 @@
+/*
+* Copyright 2020-2023 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package csidriver
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"github.com/pkg/errors"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"k8s.io/mount-utils"
+
+	"github.com/v6d-io/v6d/k8s/pkg/log"
+)
+
+// MaxVolumesPerNode is the maximum number of volumes supported per node.
+const MaxVolumesPerNode = 1000
+
+func (vc *VineyardCSI) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
+	log.Infof("NodeStageVolume: called with args %+v", *req)
+	if req.VolumeId == "" {
+		return nil, status.Error(codes.InvalidArgument, "NodeStageVolume Volume ID must be provided")
+	}
+
+	if req.StagingTargetPath == "" {
+		return nil, status.Error(codes.InvalidArgument, "NodeStageVolume Staging Target Path must be provided")
+	}
+
+	if req.VolumeCapability == nil {
+		return nil, status.Error(codes.InvalidArgument, "NodeStageVolume Volume Capability must be provided")
+	}
+	return &csi.NodeStageVolumeResponse{}, nil
+}
+
+func (vc *VineyardCSI) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) {
+	log.Infof("NodeUnstageVolume: called with args %+v", *req)
+	if req.VolumeId == "" {
+		return nil, status.Error(codes.InvalidArgument, "NodeUnstageVolume Volume ID must be provided")
+	}
+
+	if req.StagingTargetPath == "" {
+		return nil, status.Error(codes.InvalidArgument, "NodeUnstageVolume Staging Target Path must be provided")
+	}
+
+	return &csi.NodeUnstageVolumeResponse{}, nil
+}
+
+func (vc *VineyardCSI) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
+	log.Infof("NodePublishVolume: called with args %+v", *req)
+
+	// Check arguments
+	if req.GetVolumeCapability() == nil {
+		return nil, status.Error(codes.InvalidArgument, "Volume capability missing in request")
+	}
+	if len(req.GetVolumeId()) == 0 {
+		return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request")
+	}
+	if len(req.GetTargetPath()) == 0 {
+		return nil, status.Error(codes.InvalidArgument, "Target path missing in request")
+	}
+	log.Infof("args check success")
+
+	volumeContext := req.VolumeContext
+	vineyardNamespace, ok := volumeContext[VineyardNamespaceKey]
+	if !ok {
+		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("vineyard namespace %s not found", VineyardNamespaceKey))
+	}
+	vineyardName, ok := volumeContext[VineyardNameKey]
+	if !ok {
+		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("vineyard name %s not found", VineyardNameKey))
+	}
+
+	vineyardSocketDir := filepath.Join(VineyardSocketPrefix, vineyardNamespace, vineyardName)
+	socket := filepath.Join(vineyardSocketDir, VineyardSocket)
+	if _, err := os.Stat(socket); err != nil {
+		return nil, status.Error(codes.Internal, fmt.Sprintf("vineyard socket %s not found: %v", socket, err))
+	}
+
+	// check if target directory exists, create if not
+	targetPath := req.GetTargetPath()
+	log.Infof("target path: %s", targetPath)
+	_, err := os.Stat(targetPath)
+	if errors.Is(err, os.ErrNotExist) {
+		if err := os.MkdirAll(targetPath, 0750); err != nil {
+			return nil, status.Error(codes.Internal, fmt.Sprintf("create directory path {%s} error: %v", targetPath, err))
+		}
+	} else if err != nil {
+		return nil, status.Error(codes.Internal, fmt.Sprintf("get target path {%s} failed: %v", targetPath, err))
+	}
+
+	mounter := mount.New("")
+	options := []string{"bind"}
+	if err := mounter.Mount(vineyardSocketDir, targetPath, "", options); err != nil {
+		return nil, status.Error(codes.Internal, fmt.Sprintf("mount %s to %s failed: %v", vineyardSocketDir, targetPath, err))
+	}
+	log.Infof("mount %s to %s success", vineyardSocketDir, req.TargetPath)
+
+	return &csi.NodePublishVolumeResponse{}, nil
+}
+
+func (vc *VineyardCSI) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) {
+	log.Infof("NodeUnpublishVolume: called with args %+v", *req)
+	if req.VolumeId == "" {
+		return nil, status.Error(codes.InvalidArgument, "NodeUnpublishVolume Volume ID must be provided")
+	}
+
+	if req.TargetPath == "" {
+		return nil, status.Error(codes.InvalidArgument, "NodeUnpublishVolume Target Path must be provided")
+	}
+
+	targetPath := req.GetTargetPath()
+	if err := mount.CleanupMountPoint(targetPath, mount.New(""), true); err != nil {
+		return nil, status.Error(codes.Internal, fmt.Sprintf("unmount %s failed: %v", targetPath, err))
+	}
+	log.Infof("unmount %s success", targetPath)
+
+	return &csi.NodeUnpublishVolumeResponse{}, nil
+}
+
+func (vc *VineyardCSI) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) {
+	log.Infof("NodeGetInfo: called with args %+v", *req)
+
+	return &csi.NodeGetInfoResponse{
+		NodeId:            vc.nodeID,
+		MaxVolumesPerNode: MaxVolumesPerNode,
+		AccessibleTopology: &csi.Topology{
+			Segments: map[string]string{
+				"kubernetes.io/hostname": vc.nodeID,
+			},
+		},
+	}, nil
+}
+
+func (vc *VineyardCSI) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) {
+	log.Infof("NodeGetCapabilities: called with args %+v", *req)
+
+	return &csi.NodeGetCapabilitiesResponse{
+		Capabilities: []*csi.NodeServiceCapability{
+			{
+				Type: &csi.NodeServiceCapability_Rpc{
+					Rpc: &csi.NodeServiceCapability_RPC{
+						Type: csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME,
+					},
+				},
+			},
+		},
+	}, nil
+}
+
+func (vc *VineyardCSI) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) {
+	if req.VolumeId == "" {
+		return nil, status.Error(codes.InvalidArgument, "NodeGetVolumeStats Volume ID must be provided")
+	}
+	return nil, status.Error(codes.Unimplemented, "")
+}
+
+func (vc *VineyardCSI) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) {
+	if len(req.GetVolumeId()) == 0 {
+		return nil, status.Error(codes.InvalidArgument, "NodeExpandVolume volume ID not provided")
+	}
+
+	if len(req.GetVolumePath()) == 0 {
+		return nil, status.Error(codes.InvalidArgument, "NodeExpandVolume volume path not provided")
+	}
+	return nil, status.Error(codes.Unimplemented, "")
+}
diff --git a/k8s/pkg/csidriver/state.go b/k8s/pkg/csidriver/state.go
new file mode 100644
index 00000000..8336956c
--- /dev/null
+++ b/k8s/pkg/csidriver/state.go
@@ -0,0 +1,127 @@
+/*
+* Copyright 2020-2023 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package csidriver
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+type Volume struct {
+	VolName        string
+	VolID          string
+	VolSize        int64
+	VolPath        string
+	NodeID         string
+	ReadOnlyAttach bool
+	Attched        bool
+}
+
+type VolumeStates struct {
+	Volumes       []Volume
+	StateFilePath string
+}
+
+func NewVolumeStates(statefilePath string) (*VolumeStates, error) {
+	s := &VolumeStates{
+		StateFilePath: statefilePath,
+	}
+
+	return s, s.restore()
+}
+
+func (s *VolumeStates) dump() error {
+	data, err := json.Marshal(&s.Volumes)
+	if err != nil {
+		return status.Errorf(codes.Internal, "error encoding volumes and snapshots: %v", err)
+	}
+	if err := os.WriteFile(s.StateFilePath, data, 0600); err != nil {
+		return status.Errorf(codes.Internal, "error writing state file: %v", err)
+	}
+	return nil
+}
+
+func (s *VolumeStates) restore() error {
+	s.Volumes = nil
+
+	data, err := os.ReadFile(s.StateFilePath)
+	switch {
+	case errors.Is(err, os.ErrNotExist):
+		// create the /csi directory if it does not exist
+		if err := os.MkdirAll(filepath.Dir(s.StateFilePath), 0750); err != nil {
+			return status.Errorf(codes.Internal, "error creating state file directory: %v", err)
+		}
+		// create the state file if it does not exist
+		if _, err := os.Create(s.StateFilePath); err != nil {
+			return status.Errorf(codes.Internal, "error creating state file: %v", err)
+		}
+		return nil
+	case err != nil:
+		return status.Errorf(codes.Internal, "error reading state file: %v", err)
+	}
+	if len(data) == 0 {
+		s.Volumes = []Volume{}
+		return nil
+	}
+	if err := json.Unmarshal(data, &s.Volumes); err != nil {
+		return status.Errorf(codes.Internal, "error encoding volumes and snapshots from state file %q: %v", s.StateFilePath, err)
+	}
+	return nil
+}
+
+func (s *VolumeStates) GetVolumeByID(volID string) (Volume, error) {
+	for _, volume := range s.Volumes {
+		if volume.VolID == volID {
+			return volume, nil
+		}
+	}
+	return Volume{}, status.Errorf(codes.NotFound, "volume id %s does not exist in the volumes list", volID)
+}
+
+func (s *VolumeStates) GetVolumeByName(volName string) (Volume, error) {
+	for _, volume := range s.Volumes {
+		if volume.VolName == volName {
+			return volume, nil
+		}
+	}
+	return Volume{}, status.Errorf(codes.NotFound, "volume name %s does not exist in the volumes list", volName)
+}
+
+func (s *VolumeStates) AddVolume(v Volume) error {
+	for i, volume := range s.Volumes {
+		if volume.VolID == v.VolID {
+			s.Volumes[i] = v
+			return nil
+		}
+	}
+	s.Volumes = append(s.Volumes, v)
+	return s.dump()
+}
+
+func (s *VolumeStates) DeleteVolume(volID string) error {
+	for i, volume := range s.Volumes {
+		if volume.VolID == volID {
+			s.Volumes = append(s.Volumes[:i], s.Volumes[i+1:]...)
+			return s.dump()
+		}
+	}
+	return nil
+}
diff --git a/k8s/pkg/schedulers/scheduling_strategy.go b/k8s/pkg/schedulers/scheduling_strategy.go
index 140a11ea..b8c8c0b7 100644
--- a/k8s/pkg/schedulers/scheduling_strategy.go
+++ b/k8s/pkg/schedulers/scheduling_strategy.go
@@ -131,6 +131,7 @@ func (b *BestEffortStrategy) TrackingChunksByCRD() *BestEffortStrategy {
 	}
 	locations := b.GetLocationsByLocalObject(localObjects)
 	nchunks, nodes := b.GetObjectInfo(locations, len(localObjects), b.replica)
+
 	b.locations = locations
 	b.nchunks = nchunks
 	b.nodes = nodes
diff --git a/k8s/pkg/templates/csidriver/daemonset.yaml b/k8s/pkg/templates/csidriver/daemonset.yaml
new file mode 100644
index 00000000..8e739cea
--- /dev/null
+++ b/k8s/pkg/templates/csidriver/daemonset.yaml
@@ -0,0 +1,112 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: {{ .Name }}-csi-nodes
+  namespace: {{ .Namespace }}
+spec:
+  selector:
+    matchLabels:
+      app: {{ .Name }}-csi-nodes
+  template:
+    metadata:
+      labels:
+        app: {{ .Name }}-csi-nodes
+        app.vineyard.io/name: {{ .Name }}
+        app.vineyard.io/role: csi-nodes
+    spec:
+      containers:
+      - command:
+        - /vineyardctl
+        args:
+        - csi
+        - --endpoint=$(CSI_ENDPOINT)
+        - --nodeid=$(KUBE_NODE_NAME)
+        - --state-file-path=/csi/state
+        {{ if .Spec.EnableDebugLog }}
+        - --verbose
+        {{ end }}
+        env:
+        - name: CSI_ENDPOINT
+          value: unix:///csi/csi.sock
+        - name: KUBE_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        image: {{ .Spec.Image }}
+        imagePullPolicy: {{ .Spec.ImagePullPolicy }}
+        name: vineyard-csi-driver
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /var/lib/kubelet
+          mountPropagation: Bidirectional
+          name: kubelet-dir
+        - mountPath: /csi
+          name: plugin-dir
+        - mountPath: /dev
+          name: device-dir
+        - mountPath: /var/run/vineyard-kubernetes
+          name: vineyard-sockets
+      - args:
+        - --csi-address=$(ADDRESS)
+        - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+        - --v=5
+        env:
+        - name: ADDRESS
+          value: /csi/csi.sock
+        - name: DRIVER_REG_SOCK_PATH
+          value: /var/lib/kubelet/plugins/csi.vineyard.v6d.io/csi.sock
+        image: {{ .Spec.Sidecar.NodeRegistrarImage }}
+        imagePullPolicy: {{ .Spec.Sidecar.ImagePullPolicy }}
+        lifecycle:
+          preStop:
+            exec:
+              command:
+              - /bin/sh
+              - -c
+              - rm -rf /registration/csi.vineyard.v6d.io-reg.sock /csi/csi.sock
+        name: node-driver-registrar
+        volumeMounts:
+        - mountPath: /csi
+          name: plugin-dir
+        - mountPath: /registration
+          name: registration-dir
+        - mountPath: /var/run/vineyard-kubernetes
+          name: vineyard-sockets
+      - args:
+        - --csi-address=/csi/csi.sock
+        - --v=5
+        image: {{ .Spec.Sidecar.LivenessProbeImage }}
+        imagePullPolicy: {{ .Spec.Sidecar.ImagePullPolicy }}
+        name: liveness-probe
+        volumeMounts:
+        - mountPath: /csi
+          name: plugin-dir
+        - mountPath: /var/run/vineyard-kubernetes
+          name: vineyard-sockets
+      {{- if .Spec.EnableToleration }}
+      tolerations:
+        - operator: Exists
+      {{- end }}
+      volumes:
+      - hostPath:
+          path: /var/lib/kubelet
+          type: Directory
+        name: kubelet-dir
+      - hostPath:
+          path: /var/lib/kubelet/plugins/csi.vineyard.v6d.io/
+          type: DirectoryOrCreate
+        name: plugin-dir
+      - hostPath:
+          path: /var/lib/kubelet/plugins_registry/
+          type: Directory
+        name: registration-dir
+      - hostPath:
+          path: /dev
+          type: Directory
+        name: device-dir
+      - hostPath:
+          path: /var/run/vineyard-kubernetes
+          type: Directory
+        name: vineyard-sockets
diff --git a/k8s/pkg/templates/csidriver/deployment.yaml b/k8s/pkg/templates/csidriver/deployment.yaml
new file mode 100644
index 00000000..55d946fa
--- /dev/null
+++ b/k8s/pkg/templates/csidriver/deployment.yaml
@@ -0,0 +1,106 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: {{ .Name }}-csi-driver
+  namespace: {{ .Namespace }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ .Name }}-csi-driver
+  template:
+    metadata:
+      labels:
+        app: {{ .Name }}-csi-driver
+        app.vineyard.io/name: {{ .Name }}
+        app.vineyard.io/role: csi-driver
+    spec:
+      containers:
+        - name: vineyard-csi-driver
+          image: {{ .Spec.Image }}
+          imagePullPolicy: {{ .Spec.ImagePullPolicy }}
+          command:
+          - /vineyardctl
+          args:
+          - csi
+          - --endpoint=$(CSI_ENDPOINT)
+          - --nodeid=$(KUBE_NODE_NAME)
+          - --state-file-path=/csi/state
+          {{ if .Spec.EnableDebugLog }}
+          - --verbose
+          {{ end }}
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: kubelet-dir
+              mountPath: /var/lib/kubelet
+              mountPropagation: "Bidirectional"
+            - name: plugin-dir
+              mountPath: /csi
+            - name: device-dir
+              mountPath: /dev
+        #Sidecar: livenessprobe
+        - name: liveness-probe
+          image: {{ .Spec.Sidecar.LivenessProbeImage }}
+          imagePullPolicy: {{ .Spec.Sidecar.ImagePullPolicy }}
+          args:
+            - "--csi-address=/csi/csi.sock"
+            - "--v=5"
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+        #Sidecar: csi-provisioner
+        - name: csi-provisioner
+          image: {{ .Spec.Sidecar.ProvisionerImage }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--v=5"
+            {{- if .Spec.Sidecar.EnableTopology }}
+            - "--feature-gates=Topology=True"
+            {{- end }}
+            - "--extra-create-metadata"
+          env:
+            - name: ADDRESS
+              value: unix:///csi/csi.sock
+          imagePullPolicy: {{ .Spec.Sidecar.ImagePullPolicy }}
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+        #Sidecar: csi-attacher
+        - name: csi-attacher
+          image: {{ .Spec.Sidecar.AttacherImage }}
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          imagePullPolicy: {{ .Spec.Sidecar.ImagePullPolicy }}
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+      volumes:
+        - name: kubelet-dir
+          hostPath:
+            path: /var/lib/kubelet
+            type: Directory
+        - name: plugin-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/csi.v6d.io/
+            type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry/
+            type: Directory
+        - name: device-dir
+          hostPath:
+            path: /dev
+            type: Directory
diff --git a/k8s/pkg/templates/csidriver/storageclass.yaml b/k8s/pkg/templates/csidriver/storageclass.yaml
new file mode 100644
index 00000000..a5024d5f
--- /dev/null
+++ b/k8s/pkg/templates/csidriver/storageclass.yaml
@@ -0,0 +1,10 @@
+{{- $storage := getStorageConfig }}
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: {{ $storage.Namespace -}}.{{- $storage.Name -}}.csi
+provisioner: csi.vineyard.v6d.io
+parameters:
+  k8s.v6d.io/vineyard/namespace: {{ $storage.Namespace }}
+  k8s.v6d.io/vineyard/name: {{ $storage.Name }}
+volumeBindingMode: {{ $storage.VolumeBindingMode }}
diff --git a/k8s/pkg/templates/template.go b/k8s/pkg/templates/template.go
index 187c991c..032d646e 100644
--- a/k8s/pkg/templates/template.go
+++ b/k8s/pkg/templates/template.go
@@ -23,7 +23,7 @@ import (
 	"github.com/pkg/errors"
 )
 
-//go:embed vineyardd etcd operation sidecar backup recover certmanager
+//go:embed vineyardd etcd operation sidecar backup recover certmanager csidriver
 var fs embed.FS
 
 // ReadFile reads a file from the embed.FS
diff --git a/k8s/test/csidriver/Dockerfile b/k8s/test/csidriver/Dockerfile
new file mode 100644
index 00000000..f7f6d8f4
--- /dev/null
+++ b/k8s/test/csidriver/Dockerfile
@@ -0,0 +1,25 @@
+# Build the manager binary
+FROM golang:1.19-buster as builder
+
+ENV GO111MODULE=on
+ENV CGO_ENABLED=0
+ENV GOOS=linux
+
+WORKDIR /workspace
+
+# Copy the vineyard Go SDK
+COPY go/ go/
+
+# Copy the Go source for vineyardctl
+COPY k8s/ k8s/
+
+WORKDIR /workspace/k8s
+
+RUN go mod download 
+
+# Update the working directory
+WORKDIR /workspace/k8s/cmd/commands/csi
+
+USER root
+
+CMD ["go", "test", "-run", "./..."]
diff --git a/k8s/test/csidriver/Makefile b/k8s/test/csidriver/Makefile
new file mode 100644
index 00000000..bb59ebf3
--- /dev/null
+++ b/k8s/test/csidriver/Makefile
@@ -0,0 +1,32 @@
+# Copyright 2020-2023 Alibaba Group Holding Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+VERSION ?= latest
+
+REGISTRY	:= vineyardcloudnative
+
+VINEYARD_CSI_TEST_IMAGE ?= $(REGISTRY)/vineyard-csi-driver-test:$(VERSION)
+
+.PHONY: csidriver-test
+csidriver-test:
+	cd ../../../ && \
+	if docker build --help | grep -q load; then \
+		docker build --load -f k8s/test/csidriver/Dockerfile . \
+		-t $(VINEYARD_CSI_TEST_IMAGE); \
+	else \
+		docker build -f k8s/test/csidriver/Dockerfile . \
+		-t $(VINEYARD_CSI_TEST_IMAGE); \
+	fi && \
+	docker run --rm -it --privileged=true $(VINEYARD_CSI_TEST_IMAGE)
diff --git a/k8s/test/e2e/Makefile b/k8s/test/e2e/Makefile
index b36e82f7..339b01e0 100644
--- a/k8s/test/e2e/Makefile
+++ b/k8s/test/e2e/Makefile
@@ -407,7 +407,8 @@ unittest: prepare-e2e-test build-local-cluster load-vineyardd-image
 	@make -C ${E2E_DIR} install-vineyard-operator
 	@echo "Running vineyardctl unit test.." 
 # run all unit test except deploy and scheduler in parallel
-	@export KUBECONFIG=/tmp/e2e-k8s.config && cd ${CMD_DIR}/commands && go test `go list ./... | grep -Ev "deploy|schedule"`
+# dsiable vineyard csi driver test here
+	@export KUBECONFIG=/tmp/e2e-k8s.config && cd ${CMD_DIR}/commands && go test `go list ./... | grep -Ev "deploy|schedule|csi"`
 # deploy test
 	@export KUBECONFIG="/tmp/e2e-k8s.config" && cd ${CMD_DIR}/commands/deploy && \
 		go test -run first && go test -run second && go test -run third

From 919c69d17ca236ee6e8fb8fa3c784162315d2180 Mon Sep 17 00:00:00 2001
From: Tao He <linzhu.ht@alibaba-inc.com>
Date: Tue, 19 Sep 2023 17:37:13 +0800
Subject: [PATCH 07/22] Add our new maintainer: @lidh15 to the maintainers list

Signed-off-by: Tao He <linzhu.ht@alibaba-inc.com>
---
 MAINTAINERS.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/MAINTAINERS.md b/MAINTAINERS.md
index 4f942f30..5827c64c 100644
--- a/MAINTAINERS.md
+++ b/MAINTAINERS.md
@@ -16,6 +16,7 @@ maintainers are listed in the [GOVERNANCE.md](GOVERNANCE.md) file.
 | Siyuan Zhang | [siyuan0322](https://github.com/siyuan0322)     | Alibaba     | [siyuanzhang.zsy@alibaba-inc.com](mailto:siyuanzhang.zsy@alibaba-inc.com) |
 | Diwen Zhu    | [andydiwenzhu](https://github.com/andydiwenzhu) | Alibaba     | [diwen.zdw@alibaba-inc.com](mailto:diwen.zdw@alibaba-inc.com)             |
 | Shumin Yuan  | [vegetableysm](https://github.com/vegetableysm) | Alibaba     | [yuanshumin.ysm@alibaba-inc.com](mailto:yuanshumin.ysm@alibaba-inc.com)   |
+| Denghao Li  | [lidh15](https://github.com/lidh15)              | PingAn Tech | [lidhrandom@gmail.com](mailto:lidhrandom@gmail.com)                       |
 
 ## Project Committers
 

From b2f192de5f6acf6a374c929ac420be5264220b14 Mon Sep 17 00:00:00 2001
From: Tao He <linzhu.ht@alibaba-inc.com>
Date: Tue, 19 Sep 2023 17:39:33 +0800
Subject: [PATCH 08/22] Bump up the libgrape-lite submodule version to v0.3.4

Signed-off-by: Tao He <linzhu.ht@alibaba-inc.com>
---
 modules/graph/thirdparty/libgrape-lite | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/graph/thirdparty/libgrape-lite b/modules/graph/thirdparty/libgrape-lite
index 076f3515..5f0ac21d 160000
--- a/modules/graph/thirdparty/libgrape-lite
+++ b/modules/graph/thirdparty/libgrape-lite
@@ -1 +1 @@
-Subproject commit 076f35155ee6e80adba131a1245496faa9e24846
+Subproject commit 5f0ac21d0c9bfe70176abe7038019cab8832019b

From a74e5b2a676a92dc4313fb6cba9ca3bbc6c123bc Mon Sep 17 00:00:00 2001
From: Tao He <linzhu.ht@alibaba-inc.com>
Date: Tue, 19 Sep 2023 18:30:23 +0800
Subject: [PATCH 09/22] Fixes the vineyard.io.serialize failure on CI (#1570)

Fixes #1569

Signed-off-by: Tao He <linzhu.ht@alibaba-inc.com>
---
 python/vineyard/drivers/io/adaptors/write_bytes.py |  9 ++++++++-
 .../drivers/io/adaptors/write_bytes_collection.py  | 14 ++++++++++++--
 python/vineyard/drivers/io/adaptors/write_orc.py   |  8 +++++++-
 .../vineyard/drivers/io/adaptors/write_parquet.py  |  8 +++++++-
 4 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/python/vineyard/drivers/io/adaptors/write_bytes.py b/python/vineyard/drivers/io/adaptors/write_bytes.py
index 0d967942..05766bda 100755
--- a/python/vineyard/drivers/io/adaptors/write_bytes.py
+++ b/python/vineyard/drivers/io/adaptors/write_bytes.py
@@ -22,6 +22,7 @@
 import sys
 
 import fsspec
+from fsspec.core import get_fs_token_paths
 
 import vineyard
 from vineyard.io.byte import ByteStream
@@ -73,7 +74,13 @@ def write_bytes(
     instream: ByteStream = streams[proc_index]
     try:
         reader = instream.open_reader(client)
-        of = fsspec.open(f"{path}_{proc_index}", "wb", **storage_options)
+
+        fs, _, _ = get_fs_token_paths(
+            f"{path}_{proc_index}", storage_options=storage_options
+        )
+        if hasattr(fs, 'auto_mkdir'):
+            fs.auto_mkdir = True
+        of = fs.open(f"{path}_{proc_index}", "wb", **storage_options)
     except Exception:  # pylint: disable=broad-except
         report_exception()
         sys.exit(-1)
diff --git a/python/vineyard/drivers/io/adaptors/write_bytes_collection.py b/python/vineyard/drivers/io/adaptors/write_bytes_collection.py
index 548aac82..b54e47a5 100755
--- a/python/vineyard/drivers/io/adaptors/write_bytes_collection.py
+++ b/python/vineyard/drivers/io/adaptors/write_bytes_collection.py
@@ -27,6 +27,7 @@
 from typing import Dict
 
 import fsspec
+from fsspec.core import get_fs_token_paths
 
 import vineyard
 from vineyard._C import ObjectID
@@ -54,7 +55,10 @@ def write_metadata(streams: StreamCollection, prefix: str, storage_options: Dict
         prefix, metadata[StreamCollection.KEY_OF_PATH], 'metadata.json'
     )
     logger.info('creating metadata for %r ...', metadata_path)
-    with fsspec.open(metadata_path, 'wb', **storage_options) as fp:
+    fs, _, _ = get_fs_token_paths(metadata_path, storage_options=storage_options)
+    if hasattr(fs, 'auto_mkdir'):
+        fs.auto_mkdir = True
+    with fs.open(metadata_path, 'wb', **storage_options) as fp:
         fp.write(json.dumps(metadata).encode('utf-8'))
 
 
@@ -62,7 +66,13 @@ def write_byte_stream(client, stream: ByteStream, prefix: str, storage_options:
     path = stream.params[StreamCollection.KEY_OF_PATH]
     try:
         reader = stream.open_reader(client)
-        of = fsspec.open(os.path.join(prefix, path), "wb", **storage_options)
+
+        fs, _, _ = get_fs_token_paths(
+            os.path.join(prefix, path), storage_options=storage_options
+        )
+        if hasattr(fs, 'auto_mkdir'):
+            fs.auto_mkdir = True
+        of = fs.open(os.path.join(prefix, path), "wb", **storage_options)
     except Exception:  # pylint: disable=broad-except
         report_exception()
         sys.exit(-1)
diff --git a/python/vineyard/drivers/io/adaptors/write_orc.py b/python/vineyard/drivers/io/adaptors/write_orc.py
index a3fdddf0..34fa5e58 100755
--- a/python/vineyard/drivers/io/adaptors/write_orc.py
+++ b/python/vineyard/drivers/io/adaptors/write_orc.py
@@ -25,6 +25,7 @@
 
 import cloudpickle
 import fsspec
+from fsspec.core import get_fs_token_paths
 
 import vineyard
 from vineyard.io.dataframe import DataframeStream
@@ -59,7 +60,12 @@ def write_orc(
     chunk_hook = write_options.get('chunk_hook', None)
 
     writer = None
-    with fsspec.open(f"{path}_{proc_index}", "wb", **storage_options) as fp:
+    fs, _, _ = get_fs_token_paths(
+        f"{path}_{proc_index}", storage_options=storage_options
+    )
+    if hasattr(fs, 'auto_mkdir'):
+        fs.auto_mkdir = True
+    with fs.open(f"{path}_{proc_index}", "wb", **storage_options) as fp:
         while True:
             try:
                 batch = reader.next()
diff --git a/python/vineyard/drivers/io/adaptors/write_parquet.py b/python/vineyard/drivers/io/adaptors/write_parquet.py
index ae3f1db9..b4c45df5 100755
--- a/python/vineyard/drivers/io/adaptors/write_parquet.py
+++ b/python/vineyard/drivers/io/adaptors/write_parquet.py
@@ -23,6 +23,7 @@
 
 import cloudpickle
 import fsspec
+from fsspec.core import get_fs_token_paths
 
 import vineyard
 from vineyard.io.dataframe import DataframeStream
@@ -57,7 +58,12 @@ def write_parquet(
     chunk_hook = write_options.get('chunk_hook', None)
 
     writer = None
-    with fsspec.open(f"{path}_{proc_index}", "wb", **storage_options) as fp:
+    fs, _, _ = get_fs_token_paths(
+        f"{path}_{proc_index}", storage_options=storage_options
+    )
+    if hasattr(fs, 'auto_mkdir'):
+        fs.auto_mkdir = True
+    with fs.open(f"{path}_{proc_index}", "wb", **storage_options) as fp:
         while True:
             try:
                 batch = reader.next()

From 8a73f06aa2a912fcbb7a1560cfc25554d55e100e Mon Sep 17 00:00:00 2001
From: Tao He <linzhu.ht@alibaba-inc.com>
Date: Tue, 19 Sep 2023 18:31:53 +0800
Subject: [PATCH 10/22] Bump up vineyard version to v0.17.2

Signed-off-by: Tao He <linzhu.ht@alibaba-inc.com>
---
 CMakeLists.txt                      | 2 +-
 charts/vineyard-operator/Chart.yaml | 4 ++--
 k8s/cmd/main.go                     | 2 +-
 setup.cfg                           | 2 +-
 src/common/util/config.h            | 4 ++--
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 57402259..d16cae1e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.3)
 
 set(VINEYARD_MAJOR_VERSION 0)
 set(VINEYARD_MINOR_VERSION 17)
-set(VINEYARD_PATCH_VERSION 1)
+set(VINEYARD_PATCH_VERSION 2)
 set(VINEYARD_VERSION ${VINEYARD_MAJOR_VERSION}.${VINEYARD_MINOR_VERSION}.${VINEYARD_PATCH_VERSION})
 
 message(STATUS "Configuring and building vineyard version '${VINEYARD_VERSION}'.")
diff --git a/charts/vineyard-operator/Chart.yaml b/charts/vineyard-operator/Chart.yaml
index 551863ea..c6768975 100644
--- a/charts/vineyard-operator/Chart.yaml
+++ b/charts/vineyard-operator/Chart.yaml
@@ -24,12 +24,12 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.17.1
+version: 0.17.2
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: 0.17.1
+appVersion: 0.17.2
 
 dependencies:
   - name: cert-manager
diff --git a/k8s/cmd/main.go b/k8s/cmd/main.go
index e291d558..46954bd2 100644
--- a/k8s/cmd/main.go
+++ b/k8s/cmd/main.go
@@ -48,7 +48,7 @@ var cmdLong = util.LongDesc(`
 
 var cmd = &cobra.Command{
 	Use:     "vineyardctl [command]",
-	Version: "v0.17.1",
+	Version: "v0.17.2",
 	Short:   "vineyardctl is the command-line tool for interact with the Vineyard Operator.",
 	Long:    cmdLong,
 }
diff --git a/setup.cfg b/setup.cfg
index fa379df5..23a7c698 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
 [metadata]
-version = 0.17.1
+version = 0.17.2
 
 [pycodestyle]
 max_line_length = 88
diff --git a/src/common/util/config.h b/src/common/util/config.h
index e904ece0..3dddcd44 100644
--- a/src/common/util/config.h
+++ b/src/common/util/config.h
@@ -18,11 +18,11 @@ limitations under the License.
 
 #define VINEYARD_VERSION_MAJOR 0
 #define VINEYARD_VERSION_MINOR 17
-#define VINEYARD_VERSION_PATCH 1
+#define VINEYARD_VERSION_PATCH 2
 
 #define VINEYARD_VERSION                                              \
   ((VINEYARD_VERSION_MAJOR * 1000) + VINEYARD_VERSION_MINOR) * 1000 + \
       VINEYARD_VERSION_PATCH
-#define VINEYARD_VERSION_STRING "0.17.1"
+#define VINEYARD_VERSION_STRING "0.17.2"
 
 #endif  // SRC_COMMON_UTIL_CONFIG_H_

From 08f9a87e664d4523975551c9808cbb1d1dc9c3bb Mon Sep 17 00:00:00 2001
From: Ye Cao <caoye.cao@alibaba-inc.com>
Date: Tue, 19 Sep 2023 20:43:54 +0800
Subject: [PATCH 11/22] Delete the latest tag so that the helm chart use the
 Chart.AppVersion as the image tag (#1572)

Fixes #1571

Signed-off-by: Ye Cao <caoye.cao@alibaba-inc.com>
---
 charts/vineyard-operator/values.yaml | 1 -
 k8s/Makefile                         | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/vineyard-operator/values.yaml b/charts/vineyard-operator/values.yaml
index e743095d..75154612 100644
--- a/charts/vineyard-operator/values.yaml
+++ b/charts/vineyard-operator/values.yaml
@@ -23,7 +23,6 @@ controllerManager:
   manager:
     image:
       repository: vineyardcloudnative/vineyard-operator
-      tag: latest
     imagePullPolicy: IfNotPresent
     resources:
       limits:
diff --git a/k8s/Makefile b/k8s/Makefile
index ab232e5e..ad2b09ad 100644
--- a/k8s/Makefile
+++ b/k8s/Makefile
@@ -287,5 +287,6 @@ generate-helm-chart: helmify kustomize
 	cd ../charts && $(KUSTOMIZE) build ../k8s/config/default | $(HELMIFY) --cert-manager-as-subchart vineyard-operator && \
 	sed -i 's/\/var\/run\/vineyard-kubernetes\/{{.Namespace}}\/{{.Name}}/\/var\/run\/vineyard-kubernetes\/{{ \"{{.Namespace}}\/{{.Name}}\" }}/g' \
 	vineyard-operator/templates/vineyardd-crd.yaml && \
+	sed -i '/tag: latest/d' vineyard-operator/values.yaml && \
 	sed -i 's/certManager/cert-manager/g' vineyard-operator/values.yaml && \
 	sed -i '4i\  extraArgs:\n    - --enable-certificate-owner-ref=true' vineyard-operator/values.yaml

From 857ae4852895fba197174ead9fb1f19c5eadec0b Mon Sep 17 00:00:00 2001
From: Tao He <linzhu.ht@alibaba-inc.com>
Date: Tue, 19 Sep 2023 20:45:01 +0800
Subject: [PATCH 12/22] Bump up the etcd-cpp-apiv3 version to fixes the
 protobuf error

Signed-off-by: Tao He <linzhu.ht@alibaba-inc.com>
---
 thirdparty/etcd-cpp-apiv3 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/thirdparty/etcd-cpp-apiv3 b/thirdparty/etcd-cpp-apiv3
index 15c022e3..6fc1f164 160000
--- a/thirdparty/etcd-cpp-apiv3
+++ b/thirdparty/etcd-cpp-apiv3
@@ -1 +1 @@
-Subproject commit 15c022e36c61e20aec6bcdfc5b34adc5273f798a
+Subproject commit 6fc1f164c0495baab810b358077c49e088d22034

From 11dfc197adfb185631db20cf5da20f22c283d2fc Mon Sep 17 00:00:00 2001
From: Tao He <linzhu.ht@alibaba-inc.com>
Date: Wed, 20 Sep 2023 11:02:08 +0800
Subject: [PATCH 13/22] Rust: bump up version and allow a range of version for
 deps (#1574)

Signed-off-by: Tao He <linzhu.ht@alibaba-inc.com>
---
 rust/Cargo.toml |  28 ++---
 rust/deny.toml  | 276 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 290 insertions(+), 14 deletions(-)
 create mode 100644 rust/deny.toml

diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index 507a2864..caef8173 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -8,7 +8,7 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "0.17.1"
+version = "0.17.2"
 homepage = "https://v6d.io"
 repository = "https://github.com/v6d-io/v6d.git"
 authors = ["Vineyard <vineyard@alibaba-inc.com>"]
@@ -22,13 +22,13 @@ edition = "2021"
 readme = "README.md"
 
 [workspace.dependencies]
-arrow-array = ">=40, <44"
-arrow-buffer = ">=40, <44"
-arrow-ipc = ">=40, <44"
-arrow-schema = ">=40, <44"
-arrow2 = { version = "0.17", features = ["arrow"] }
+arrow-array = ">=40, <47"
+arrow-buffer = ">=40, <47"
+arrow-ipc = ">=40, <47"
+arrow-schema = ">=40, <47"
+arrow2 = { version = ">=0.17, <0.19", features = ["arrow"] }
 ctor = "0.2"
-datafusion = "28"
+datafusion = ">= 28"
 downcast-rs = "1.2"
 env_logger = "0.9"
 futures = "0.3"
@@ -38,10 +38,10 @@ itertools = "0.11"
 lazy_static = "1"
 log = "0.4"
 memmap2 = "0.7"
-num-traits = "0.2"
-num-derive = "0.4"
+num-traits = ">= 0.2"
+num-derive = ">= 0.4"
 parking_lot = "0.12"
-polars-core = "0.32"
+polars-core = ">=0.32, <34"
 rand = "0.8"
 sendfd = "0.4"
 serde = "1.0"
@@ -50,7 +50,7 @@ serde_json = "1.0"
 spectral = "0.6"
 static_str_ops = "0.1.2"
 
-vineyard = { version = "0.17.1", path = "./vineyard" }
-vineyard-datafusion = { version = "0.17.1", path = "./vineyard-datafusion" }
-vineyard-polars = { version = "0.17.1", path = "./vineyard-polars" }
-vineyard-integration-testing = { version = "0.17.1", path = "./vineyard-integration-testing" }
+vineyard = { version = "0.17.2", path = "./vineyard" }
+vineyard-datafusion = { version = "0.17.2", path = "./vineyard-datafusion" }
+vineyard-polars = { version = "0.17.2", path = "./vineyard-polars" }
+vineyard-integration-testing = { version = "0.17.2", path = "./vineyard-integration-testing" }
diff --git a/rust/deny.toml b/rust/deny.toml
new file mode 100644
index 00000000..bccefca3
--- /dev/null
+++ b/rust/deny.toml
@@ -0,0 +1,276 @@
+# This template contains all of the possible sections and their default values
+
+# Note that all fields that take a lint level have these possible values:
+# * deny - An error will be produced and the check will fail
+# * warn - A warning will be produced, but the check will not fail
+# * allow - No warning or error will be produced, though in some cases a note
+# will be
+
+# The values provided in this template are the default values that will be used
+# when any section or field is not specified in your own configuration
+
+# Root options
+
+# If 1 or more target triples (and optionally, target_features) are specified,
+# only the specified targets will be checked when running `cargo deny check`.
+# This means, if a particular package is only ever used as a target specific
+# dependency, such as, for example, the `nix` crate only being used via the
+# `target_family = "unix"` configuration, that only having windows targets in
+# this list would mean the nix crate, as well as any of its exclusive
+# dependencies not shared by any other crates, would be ignored, as the target
+# list here is effectively saying which targets you are building for.
+targets = [
+    # The triple can be any string, but only the target triples built in to
+    # rustc (as of 1.40) can be checked against actual config expressions
+    #{ triple = "x86_64-unknown-linux-musl" },
+    # You can also specify which target_features you promise are enabled for a
+    # particular target. target_features are currently not validated against
+    # the actual valid features supported by the target architecture.
+    #{ triple = "wasm32-unknown-unknown", features = ["atomics"] },
+]
+# When creating the dependency graph used as the source of truth when checks are
+# executed, this field can be used to prune crates from the graph, removing them
+# from the view of cargo-deny. This is an extremely heavy hammer, as if a crate
+# is pruned from the graph, all of its dependencies will also be pruned unless
+# they are connected to another crate in the graph that hasn't been pruned,
+# so it should be used with care. The identifiers are [Package ID Specifications]
+# (https://doc.rust-lang.org/cargo/reference/pkgid-spec.html)
+#exclude = []
+# If true, metadata will be collected with `--all-features`. Note that this can't
+# be toggled off if true, if you want to conditionally enable `--all-features` it
+# is recommended to pass `--all-features` on the cmd line instead
+all-features = false
+# If true, metadata will be collected with `--no-default-features`. The same
+# caveat with `all-features` applies
+no-default-features = false
+# If set, these feature will be enabled when collecting metadata. If `--features`
+# is specified on the cmd line they will take precedence over this option.
+#features = []
+# When outputting inclusion graphs in diagnostics that include features, this
+# option can be used to specify the depth at which feature edges will be added.
+# This option is included since the graphs can be quite large and the addition
+# of features from the crate(s) to all of the graph roots can be far too verbose.
+# This option can be overridden via `--feature-depth` on the cmd line
+feature-depth = 1
+
+# This section is considered when running `cargo deny check advisories`
+# More documentation for the advisories section can be found here:
+# https://embarkstudios.github.io/cargo-deny/checks/advisories/cfg.html
+[advisories]
+# The path where the advisory database is cloned/fetched into
+db-path = "~/.cargo/advisory-db"
+# The url(s) of the advisory databases to use
+db-urls = ["https://github.com/rustsec/advisory-db"]
+# The lint level for security vulnerabilities
+vulnerability = "allow"
+# The lint level for unmaintained crates
+unmaintained = "warn"
+# The lint level for crates that have been yanked from their source registry
+yanked = "warn"
+# The lint level for crates with security notices. Note that as of
+# 2019-12-17 there are no security notice advisories in
+# https://github.com/rustsec/advisory-db
+notice = "warn"
+# A list of advisory IDs to ignore. Note that ignored advisories will still
+# output a note when they are encountered.
+ignore = [
+    #"RUSTSEC-0000-0000",
+]
+# Threshold for security vulnerabilities, any vulnerability with a CVSS score
+# lower than the range specified will be ignored. Note that ignored advisories
+# will still output a note when they are encountered.
+# * None - CVSS Score 0.0
+# * Low - CVSS Score 0.1 - 3.9
+# * Medium - CVSS Score 4.0 - 6.9
+# * High - CVSS Score 7.0 - 8.9
+# * Critical - CVSS Score 9.0 - 10.0
+#severity-threshold =
+
+# If this is true, then cargo deny will use the git executable to fetch advisory database.
+# If this is false, then it uses a built-in git library.
+# Setting this to true can be helpful if you have special authentication requirements that cargo-deny does not support.
+# See Git Authentication for more information about setting up git authentication.
+#git-fetch-with-cli = true
+
+# This section is considered when running `cargo deny check licenses`
+# More documentation for the licenses section can be found here:
+# https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html
+[licenses]
+# The lint level for crates which do not have a detectable license
+unlicensed = "deny"
+# List of explicitly allowed licenses
+# See https://spdx.org/licenses/ for list of possible licenses
+# [possible values: any SPDX 3.11 short identifier (+ optional exception)].
+allow = [
+    #"MIT",
+    #"Apache-2.0",
+    #"Apache-2.0 WITH LLVM-exception",
+]
+# List of explicitly disallowed licenses
+# See https://spdx.org/licenses/ for list of possible licenses
+# [possible values: any SPDX 3.11 short identifier (+ optional exception)].
+deny = [
+    #"Nokia",
+]
+# Lint level for licenses considered copyleft
+copyleft = "allow"
+# Blanket approval or denial for OSI-approved or FSF Free/Libre licenses
+# * both - The license will be approved if it is both OSI-approved *AND* FSF
+# * either - The license will be approved if it is either OSI-approved *OR* FSF
+# * osi - The license will be approved if it is OSI approved
+# * fsf - The license will be approved if it is FSF Free
+# * osi-only - The license will be approved if it is OSI-approved *AND NOT* FSF
+# * fsf-only - The license will be approved if it is FSF *AND NOT* OSI-approved
+# * neither - This predicate is ignored and the default lint level is used
+allow-osi-fsf-free = "neither"
+# Lint level used when no other predicates are matched
+# 1. License isn't in the allow or deny lists
+# 2. License isn't copyleft
+# 3. License isn't OSI/FSF, or allow-osi-fsf-free = "neither"
+default = "allow"
+# The confidence threshold for detecting a license from license text.
+# The higher the value, the more closely the license text must be to the
+# canonical license text of a valid SPDX license file.
+# [possible values: any between 0.0 and 1.0].
+confidence-threshold = 0.8
+# Allow 1 or more licenses on a per-crate basis, so that particular licenses
+# aren't accepted for every possible crate as with the normal allow list
+exceptions = [
+    # Each entry is the crate and version constraint, and its specific allow
+    # list
+    #{ allow = ["Zlib"], name = "adler32", version = "*" },
+]
+
+# Some crates don't have (easily) machine readable licensing information,
+# adding a clarification entry for it allows you to manually specify the
+# licensing information
+#[[licenses.clarify]]
+# The name of the crate the clarification applies to
+#name = "ring"
+# The optional version constraint for the crate
+#version = "*"
+# The SPDX expression for the license requirements of the crate
+#expression = "MIT AND ISC AND OpenSSL"
+# One or more files in the crate's source used as the "source of truth" for
+# the license expression. If the contents match, the clarification will be used
+# when running the license check, otherwise the clarification will be ignored
+# and the crate will be checked normally, which may produce warnings or errors
+# depending on the rest of your configuration
+#license-files = [
+    # Each entry is a crate relative path, and the (opaque) hash of its contents
+    #{ path = "LICENSE", hash = 0xbd0eed23 }
+#]
+
+[licenses.private]
+# If true, ignores workspace crates that aren't published, or are only
+# published to private registries.
+# To see how to mark a crate as unpublished (to the official registry),
+# visit https://doc.rust-lang.org/cargo/reference/manifest.html#the-publish-field.
+ignore = false
+# One or more private registries that you might publish crates to, if a crate
+# is only published to private registries, and ignore is true, the crate will
+# not have its license(s) checked
+registries = [
+    #"https://sekretz.com/registry
+]
+
+# This section is considered when running `cargo deny check bans`.
+# More documentation about the 'bans' section can be found here:
+# https://embarkstudios.github.io/cargo-deny/checks/bans/cfg.html
+[bans]
+# Lint level for when multiple versions of the same crate are detected
+multiple-versions = "deny"
+# Lint level for when a crate version requirement is `*`
+wildcards = "allow"
+# The graph highlighting used when creating dotgraphs for crates
+# with multiple versions
+# * lowest-version - The path to the lowest versioned duplicate is highlighted
+# * simplest-path - The path to the version with the fewest edges is highlighted
+# * all - Both lowest-version and simplest-path are used
+highlight = "all"
+# The default lint level for `default` features for crates that are members of
+# the workspace that is being checked. This can be overridden by allowing/denying
+# `default` on a crate-by-crate basis if desired.
+workspace-default-features = "allow"
+# The default lint level for `default` features for external crates that are not
+# members of the workspace. This can be overridden by allowing/denying `default`
+# on a crate-by-crate basis if desired.
+external-default-features = "allow"
+# List of crates that are allowed. Use with care!
+allow = [
+    #{ name = "ansi_term", version = "=0.11.0" },
+]
+# List of crates to deny
+deny = [
+    # Each entry the name of a crate and a version range. If version is
+    # not specified, all versions will be matched.
+    #{ name = "ansi_term", version = "=0.11.0" },
+    #
+    # Wrapper crates can optionally be specified to allow the crate when it
+    # is a direct dependency of the otherwise banned crate
+    #{ name = "ansi_term", version = "=0.11.0", wrappers = [] },
+]
+
+# List of features to allow/deny
+# Each entry the name of a crate and a version range. If version is
+# not specified, all versions will be matched.
+#[[bans.features]]
+#name = "reqwest"
+# Features to not allow
+#deny = ["json"]
+# Features to allow
+#allow = [
+#    "rustls",
+#    "__rustls",
+#    "__tls",
+#    "hyper-rustls",
+#    "rustls",
+#    "rustls-pemfile",
+#    "rustls-tls-webpki-roots",
+#    "tokio-rustls",
+#    "webpki-roots",
+#]
+# If true, the allowed features must exactly match the enabled feature set. If
+# this is set there is no point setting `deny`
+#exact = true
+
+# Certain crates/versions that will be skipped when doing duplicate detection.
+skip = [
+    { name = "bitflags" },
+    { name = "itertools" },
+    { name = "strum" },
+    { name = "strum_macros" },
+    { name = "syn" },
+    #{ name = "ansi_term", version = "=0.11.0" },
+]
+# Similarly to `skip` allows you to skip certain crates during duplicate
+# detection. Unlike skip, it also includes the entire tree of transitive
+# dependencies starting at the specified crate, up to a certain depth, which is
+# by default infinite.
+skip-tree = [
+    #{ name = "ansi_term", version = "=0.11.0", depth = 20 },
+]
+
+# This section is considered when running `cargo deny check sources`.
+# More documentation about the 'sources' section can be found here:
+# https://embarkstudios.github.io/cargo-deny/checks/sources/cfg.html
+[sources]
+# Lint level for what to happen when a crate from a crate registry that is not
+# in the allow list is encountered
+unknown-registry = "warn"
+# Lint level for what to happen when a crate from a git repository that is not
+# in the allow list is encountered
+unknown-git = "warn"
+# List of URLs for allowed crate registries. Defaults to the crates.io index
+# if not specified. If it is specified but empty, no registries are allowed.
+allow-registry = ["https://github.com/rust-lang/crates.io-index"]
+# List of URLs for allowed Git repositories
+allow-git = []
+
+[sources.allow-org]
+# 1 or more github.com organizations to allow git sources for
+github = []
+# 1 or more gitlab.com organizations to allow git sources for
+gitlab = []
+# 1 or more bitbucket.org organizations to allow git sources for
+bitbucket = []

From af6754659f22e112526021d5e41d54f050d57900 Mon Sep 17 00:00:00 2001
From: Ye Cao <caoye.cao@alibaba-inc.com>
Date: Wed, 20 Sep 2023 11:34:11 +0800
Subject: [PATCH 14/22] Fix the error in the helm chart e2e test and the sha
 not exposed in the push event (#1573)

* Use the latest tag for the vineyard operator image in the generated helm chart e2e test.
* Fix the sha not exposed in the push event.

Signed-off-by: Ye Cao <caoye.cao@alibaba-inc.com>
---
 .github/workflows/vineyard-operator.yaml       | 7 ++-----
 k8s/test/e2e/autogenerated-helm-chart/e2e.yaml | 3 ++-
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/vineyard-operator.yaml b/.github/workflows/vineyard-operator.yaml
index 5b385290..3045238e 100644
--- a/.github/workflows/vineyard-operator.yaml
+++ b/.github/workflows/vineyard-operator.yaml
@@ -297,11 +297,8 @@ jobs:
       - name: Leave a message for logs URL
         if: ${{ failure() }}
         run: |
-          if [[ -n ${{ github.event.pull_request.head.sha }} ]]; then
-              short_sha=$(git rev-parse --short ${{ github.event.pull_request.head.sha }})
-          else
-              short_sha=$(git rev-parse --short ${{ github.event.push.head.sha }})
-          fi
+          sha=${{ github.event.pull_request.head.sha || github.sha }}
+          short_sha=$(git rev-parse --short $sha)
           case ${{ matrix.job }} in
             e2e-tests-spill)
               signed_url_key=0707
diff --git a/k8s/test/e2e/autogenerated-helm-chart/e2e.yaml b/k8s/test/e2e/autogenerated-helm-chart/e2e.yaml
index d2e0f104..cd545260 100644
--- a/k8s/test/e2e/autogenerated-helm-chart/e2e.yaml
+++ b/k8s/test/e2e/autogenerated-helm-chart/e2e.yaml
@@ -27,7 +27,8 @@ setup:
         helm install vineyard-operator charts/vineyard-operator \
             --create-namespace \
             --namespace vineyard-system \
-            --set controllerManager.manager.image.repository=localhost:5001/vineyard-operator
+            --set controllerManager.manager.image.repository=localhost:5001/vineyard-operator \
+            --set controllerManager.manager.image.tag=latest
       wait:
         - namespace: vineyard-system
           resource: deployment/vineyard-operator-controller-manager

From 646c7ced641f84f587f6c7a0a09881e82f23153a Mon Sep 17 00:00:00 2001
From: Ye Cao <caoye.cao@alibaba-inc.com>
Date: Wed, 20 Sep 2023 18:02:02 +0800
Subject: [PATCH 15/22] Improve the csi driver tutorial (#1575)

Signed-off-by: Ye Cao <caoye.cao@alibaba-inc.com>
---
 ...g-in-kubeflow-with-vineyard-csi-driver.rst | 54 +++++++++----------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/docs/tutorials/kubernetes/efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst b/docs/tutorials/kubernetes/efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst
index b8305455..86c7b8e3 100644
--- a/docs/tutorials/kubernetes/efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst
+++ b/docs/tutorials/kubernetes/efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst
@@ -1,5 +1,5 @@
 Efficient data sharing in Kubeflow with Vineyard CSI Driver
-===========================================================
+-----------------------------------------------------------
 
 If you are using `Kubeflow Pipeline`_ or `Argo Workflow`_ to manage your machine learning workflow, 
 you may find that the data saving/loading to the volumes is slow.
@@ -8,7 +8,7 @@ map each vineyard object to a volume, and the data saving/loading is handled by
 Next, we will show you how to use the Vineyard CSI Driver to speed up a kubeflow pipeline.
 
 Prerequisites
--------------
+=============
 
 - A kubernetes cluster with version >= 1.25.10. If you don't have one by hand, you can refer to the 
 guide `Initialize Kubernetes Cluster`_ to create one.
@@ -16,7 +16,7 @@ guide `Initialize Kubernetes Cluster`_ to create one.
 - Install the `Argo Workflow CLI`_ by following the official guide.
 
 Deploy the Vineyard Cluster
----------------------------
+===========================
 
 .. code:: bash
 
@@ -36,7 +36,7 @@ You can check as follows:
     vineyardd-sample-etcd-0                          1/1     Running   0          4d3h
 
 Deploy the Vineyard CSI Driver
-------------------------------
+==============================
 
 Before deploying the Vineyard CSI Driver, you are supposed to check the vineyard 
 deployment is ready as follows:
@@ -71,7 +71,7 @@ Then check the status of the Vineyard CSI Driver:
     vineyardd-sample-etcd-0                          1/1     Running   0          4d3h
 
 Deploy Argo Workflows
----------------------
+=====================
 
 Install the argo server on Kubernetes:
 
@@ -90,22 +90,22 @@ Then check the status of the argo server:
     workflow-controller-b888f4458-sfrjd   1/1     Running   0          4d1h
 
 Running a Kubeflow Pipeline example
------------------------------------
+===================================
 
-The example is under the directory `k8s/examples/vineyard-csidriver`, and `pipeline.py` under this
+The example is under the directory ``k8s/examples/vineyard-csidriver``, and ``pipeline.py`` under this
 directory is the original pipeline definition. To use the Vineyard CSI Driver, we need to do two 
 modifications:
 
-1. Change APIs like `pd.read_pickle/write_pickle` to `vineyard.csi.write/read` in the source code.
+1. Change APIs like **pd.read_pickle/write_pickle** to **vineyard.csi.write/read** in the source code.
 
-2. Add the `vineyard object` VolumeOp to the pipeline's dependencies. The path in the API changed 
+2. Add the ``vineyard object`` VolumeOp to the pipeline's dependencies. The path in the API changed 
 in the first step will be mapped to a volume. Notice, the volume used in any task needs to be 
 explicitly mounted to the corresponding path in the source code, and the storageclass_name 
-format of each VolumeOp is `{vineyard-deployment-namespace}.{vineyard-deployment-name}.csi`.
+format of each VolumeOp is ``{vineyard-deployment-namespace}.{vineyard-deployment-name}.csi``.
 
-You may get some insights from the modified pipeline `pipeline-with-vineyard.py`. Then, we need to
+You may get some insights from the modified pipeline ``pipeline-with-vineyard.py``. Then, we need to
 compile the pipeline to an argo-workflow yaml. To be compatible with benchmark test, we update the
-generated `pipeline.yaml` and `pipeline-with-vineyard.yaml`.
+generated ``pipeline.yaml`` and ``pipeline-with-vineyard.yaml``.
 
 Now, we can build the docker images for the pipeline:
 
@@ -132,7 +132,7 @@ we use the kind cluster in this example, we can load the image to the clusters:
     $ make load-images
 
 To simulate the data loading/saving of the actual pipeline, we use the nfs volume
-to store the data. The nfs volume is mounted to the `/mnt/data` directory of the 
+to store the data. The nfs volume is mounted to the ``/mnt/data`` directory of the 
 kind cluster. Then apply the data volume as follows:
 
 .. tip::
@@ -173,28 +173,22 @@ Submit the kubeflow example with vineyard to the argo server:
     done
 
 Result Analysis
----------------
-
-+------------+------------------+---------------+
-| data scale | without vineyard | with vineyard |
-+============+==================+===============+
-| 8500 Mi    | 21s              | 5.4s          |
-| 12000 Mi   | 26s              | 7s            |
-| 15000 Mi   | 32.2s            | 9.4s          |
-+------------+------------------+---------------+
+===============
 
 The data scale are 8500 Mi, 12000 Mi and 15000 Mi, which correspond to 
 the 3000, 4000 and 5000 in the previous data_multiplier respectively, 
 and the time of argo workflow execution of the pipeline is as follows:
 
 Argo workflow duration
-======================
+""""""""""""""""""""""
 
 +------------+------------------+---------------+
 | data scale | without vineyard | with vineyard |
 +============+==================+===============+
 | 8500 Mi    | 186s             | 169s          |
++------------+------------------+---------------+
 | 12000 Mi   | 250s             | 203s          |
++------------+------------------+---------------+
 | 15000 Mi   | 332s             | 286s          |
 +------------+------------------+---------------+
 
@@ -208,13 +202,15 @@ with vineyard is shorter than that without vineyard.
 Also, we record the whole execution time via logs. The result is as follows:
 
 Actual execution time
-=====================
+"""""""""""""""""""""
 
 +------------+------------------+---------------+
 | data scale | without vineyard | with vineyard |
 +============+==================+===============+
 | 8500 Mi    | 139.3s           | 92.3s         |
++------------+------------------+---------------+
 | 12000 Mi   | 204.3s           | 131.1s        |
++------------+------------------+---------------+
 | 15000 Mi   | 289.3s           | 209.7s        |
 +------------+------------------+---------------+
 
@@ -224,13 +220,15 @@ execution of the pipeline with vineyard is shorter than that without vineyard.
 To be specific, we record the write/read time of the following steps:
 
 Writing time
-============
+""""""""""""
 
 +------------+------------------+---------------+
 | data scale | without vineyard | with vineyard |
 +============+==================+===============+
 | 8500 Mi    | 21s              | 5.4s          |
++------------+------------------+---------------+
 | 12000 Mi   | 26s              | 7s            |
++------------+------------------+---------------+
 | 15000 Mi   | 32.2s            | 9.4s          |
 +------------+------------------+---------------+
 
@@ -243,7 +241,7 @@ write operation of the nfs volume.
 
 
 Reading time
-============
+""""""""""""
 
 We delete the time of init data loading, and the results are as follows:
 
@@ -251,7 +249,9 @@ We delete the time of init data loading, and the results are as follows:
 | data scale | without vineyard | with vineyard |
 +============+==================+===============+
 | 8500 Mi    | 36.7s            | 0.02s         |
++------------+------------------+---------------+
 | 12000 Mi   | 45.7s            | 0.02s         |
++------------+------------------+---------------+
 | 15000 Mi   | 128.6s           | 0.04s         |
 +------------+------------------+---------------+
 
@@ -266,7 +266,7 @@ execution time of the pipeline is reduced by about 30%.
 
 
 Clean up
---------
+========
 
 Delete the rbac for the kubeflow example:
 

From c10d4b4f4b8d0ec9ac6b2a714fb1676b4b0dcc29 Mon Sep 17 00:00:00 2001
From: Ye Cao <caoye.cao@alibaba-inc.com>
Date: Thu, 21 Sep 2023 19:02:55 +0800
Subject: [PATCH 16/22] Travel the path to find the vineyard socket in the
 vineyard.csi.read/write API. (#1577)

Fixes part of #1568

Signed-off-by: Ye Cao <caoye.cao@alibaba-inc.com>
---
 python/vineyard/csi/__init__.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/python/vineyard/csi/__init__.py b/python/vineyard/csi/__init__.py
index e58fccb0..2f7b36f1 100644
--- a/python/vineyard/csi/__init__.py
+++ b/python/vineyard/csi/__init__.py
@@ -22,6 +22,16 @@
 import vineyard
 
 
+def _find_vineyard_socket(path):
+    current_dir = path
+    while current_dir != "/":
+        socket_path = os.path.join(current_dir, "vineyard.sock")
+        if os.path.exists(socket_path):
+            return socket_path
+        current_dir = os.path.dirname(current_dir)
+    return None
+
+
 def write(
     value: Any,
     path: str,
@@ -31,19 +41,17 @@ def write(
     Notice, the API is only used for CSI driver.
 
     Parameters:
-        client: IPCClient
-            The vineyard client to use.
         path: str
             The path that represents a vineyard object.
 
     .. code:: python
 
         >>> arr = np.arange(8)
-        >>> client.write(arr)
+        >>> vineyard.write(arr)
     """
-    socket_path = os.path.join(path, "vineyard.sock")
+    socket_path = _find_vineyard_socket(path)
 
-    if not os.path.exists(path):
+    if socket_path is None:
         raise FileNotFoundError(
             f"The given path is not generated by vineyard CSI driver: {path}"
         )
@@ -72,9 +80,9 @@ def read(
     Returns:
         A python object that return by the resolver, by resolving an vineyard object.
     """
-    socket_path = os.path.join(path, "vineyard.sock")
+    socket_path = _find_vineyard_socket(path)
 
-    if not os.path.exists(path):
+    if socket_path is None:
         raise FileNotFoundError(
             f"The given path is not generated by vineyard CSI driver: {path}"
         )

From 00ccfe37822f5f0f130264ca9cf339f58c5d487b Mon Sep 17 00:00:00 2001
From: Tao He <linzhu.ht@alibaba-inc.com>
Date: Thu, 21 Sep 2023 19:06:21 +0800
Subject: [PATCH 17/22] Bump up vineyard version to v0.17.3

Signed-off-by: Tao He <linzhu.ht@alibaba-inc.com>
---
 CMakeLists.txt                      | 2 +-
 charts/vineyard-operator/Chart.yaml | 4 ++--
 k8s/cmd/main.go                     | 2 +-
 setup.cfg                           | 2 +-
 src/common/util/config.h            | 4 ++--
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d16cae1e..f28102ba 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.3)
 
 set(VINEYARD_MAJOR_VERSION 0)
 set(VINEYARD_MINOR_VERSION 17)
-set(VINEYARD_PATCH_VERSION 2)
+set(VINEYARD_PATCH_VERSION 3)
 set(VINEYARD_VERSION ${VINEYARD_MAJOR_VERSION}.${VINEYARD_MINOR_VERSION}.${VINEYARD_PATCH_VERSION})
 
 message(STATUS "Configuring and building vineyard version '${VINEYARD_VERSION}'.")
diff --git a/charts/vineyard-operator/Chart.yaml b/charts/vineyard-operator/Chart.yaml
index c6768975..fdeb235c 100644
--- a/charts/vineyard-operator/Chart.yaml
+++ b/charts/vineyard-operator/Chart.yaml
@@ -24,12 +24,12 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.17.2
+version: 0.17.3
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: 0.17.2
+appVersion: 0.17.3
 
 dependencies:
   - name: cert-manager
diff --git a/k8s/cmd/main.go b/k8s/cmd/main.go
index 46954bd2..10f9c10a 100644
--- a/k8s/cmd/main.go
+++ b/k8s/cmd/main.go
@@ -48,7 +48,7 @@ var cmdLong = util.LongDesc(`
 
 var cmd = &cobra.Command{
 	Use:     "vineyardctl [command]",
-	Version: "v0.17.2",
+	Version: "v0.17.3",
 	Short:   "vineyardctl is the command-line tool for interact with the Vineyard Operator.",
 	Long:    cmdLong,
 }
diff --git a/setup.cfg b/setup.cfg
index 23a7c698..a8d9d79f 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
 [metadata]
-version = 0.17.2
+version = 0.17.3
 
 [pycodestyle]
 max_line_length = 88
diff --git a/src/common/util/config.h b/src/common/util/config.h
index 3dddcd44..66723fb8 100644
--- a/src/common/util/config.h
+++ b/src/common/util/config.h
@@ -18,11 +18,11 @@ limitations under the License.
 
 #define VINEYARD_VERSION_MAJOR 0
 #define VINEYARD_VERSION_MINOR 17
-#define VINEYARD_VERSION_PATCH 2
+#define VINEYARD_VERSION_PATCH 3
 
 #define VINEYARD_VERSION                                              \
   ((VINEYARD_VERSION_MAJOR * 1000) + VINEYARD_VERSION_MINOR) * 1000 + \
       VINEYARD_VERSION_PATCH
-#define VINEYARD_VERSION_STRING "0.17.2"
+#define VINEYARD_VERSION_STRING "0.17.3"
 
 #endif  // SRC_COMMON_UTIL_CONFIG_H_

From b8da0ca8862b8e3e5a01db69b4d99768de104a39 Mon Sep 17 00:00:00 2001
From: Ye Cao <caoye.cao@alibaba-inc.com>
Date: Thu, 21 Sep 2023 19:15:01 +0800
Subject: [PATCH 18/22] Improve the doc and fix some errors in the `vineyardctl
 create csidriver` API (#1576)

* Improve the doc.
* Change the image of Vineyard CSI Driver to `vineyard-operator` by default.
* Fix the csidriver template errors.

Signed-off-by: Ye Cao <caoye.cao@alibaba-inc.com>
---
 ...ta-sharing-in-kubeflow-with-vineyard-csi-driver.rst | 10 +++++++---
 k8s/cmd/README.md                                      |  2 +-
 k8s/cmd/commands/flags/csidriver_flags.go              |  2 +-
 k8s/pkg/templates/csidriver/daemonset.yaml             |  2 +-
 k8s/pkg/templates/csidriver/deployment.yaml            |  2 +-
 5 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/docs/tutorials/kubernetes/efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst b/docs/tutorials/kubernetes/efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst
index 86c7b8e3..675906b5 100644
--- a/docs/tutorials/kubernetes/efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst
+++ b/docs/tutorials/kubernetes/efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst
@@ -10,8 +10,7 @@ Next, we will show you how to use the Vineyard CSI Driver to speed up a kubeflow
 Prerequisites
 =============
 
-- A kubernetes cluster with version >= 1.25.10. If you don't have one by hand, you can refer to the 
-guide `Initialize Kubernetes Cluster`_ to create one.
+- A kubernetes cluster with version >= 1.25.10. If you don't have one by hand, you can refer to the guide `Initialize Kubernetes Cluster`_ to create one.
 - Install the `Vineyardctl`_ by following the official guide.
 - Install the `Argo Workflow CLI`_ by following the official guide.
 
@@ -50,6 +49,11 @@ deployment is ready as follows:
 
 Then deploy the vineyard csi driver which specifies the vineyard cluster to use:
 
+.. tip::
+
+    If you want to look into the debug logs of the vineyard csi driver, you can add a
+    flag ``--verbose`` in the following command.
+
 .. code:: bash
 
     $ vineyardctl deploy csidriver --clusters vineyard-system/vineyardd-sample
@@ -138,7 +142,7 @@ kind cluster. Then apply the data volume as follows:
 .. tip::
 
     If you already have nfs volume that can be accessed by the kubernetes cluster,
-    you can update the prepare-data.yaml to use your nfs volume.
+    you can update the ``prepare-data.yaml`` to use your nfs volume.
 
 .. code:: bash
 
diff --git a/k8s/cmd/README.md b/k8s/cmd/README.md
index 4fb33561..8bbb67ef 100644
--- a/k8s/cmd/README.md
+++ b/k8s/cmd/README.md
@@ -825,7 +825,7 @@ vineyardctl deploy csidriver [flags]
       --clusters strings                 The list of vineyard clusters.
       --enableToleration                 Enable toleration for vineyard csi driver.
   -h, --help                             help for csidriver
-  -i, --image string                     The image of vineyard csi driver. (default "vineyardcloudnative/vineyard-csi-driver")
+  -i, --image string                     The image of vineyard csi driver. (default "vineyardcloudnative/vineyard-operator")
       --imagePullPolicy string           The image pull policy of vineyard csi driver. (default "IfNotPresent")
       --livenessProbeImage string        The image of livenessProbe. (default "registry.k8s.io/sig-storage/livenessprobe:v2.8.0")
       --name string                      The name of the csi driver cr. (default "csidriver-sample")
diff --git a/k8s/cmd/commands/flags/csidriver_flags.go b/k8s/cmd/commands/flags/csidriver_flags.go
index 5b76fb7b..f168b99a 100644
--- a/k8s/cmd/commands/flags/csidriver_flags.go
+++ b/k8s/cmd/commands/flags/csidriver_flags.go
@@ -58,7 +58,7 @@ func ApplyCSIDriverClustersOpts(cmd *cobra.Command) {
 func ApplyCSIDriverOpts(cmd *cobra.Command) {
 	ApplyCSIDriverNameOpts(cmd)
 	cmd.Flags().StringVarP(&CSIDriverOpts.Image, "image", "i",
-		"vineyardcloudnative/vineyard-csi-driver",
+		"vineyardcloudnative/vineyard-operator",
 		"The image of vineyard csi driver.")
 	cmd.Flags().StringVarP(&CSIDriverOpts.ImagePullPolicy, "imagePullPolicy", "",
 		"IfNotPresent", "The image pull policy of vineyard csi driver.")
diff --git a/k8s/pkg/templates/csidriver/daemonset.yaml b/k8s/pkg/templates/csidriver/daemonset.yaml
index 8e739cea..16587ada 100644
--- a/k8s/pkg/templates/csidriver/daemonset.yaml
+++ b/k8s/pkg/templates/csidriver/daemonset.yaml
@@ -22,7 +22,7 @@ spec:
         - --endpoint=$(CSI_ENDPOINT)
         - --nodeid=$(KUBE_NODE_NAME)
         - --state-file-path=/csi/state
-        {{ if .Spec.EnableDebugLog }}
+        {{ if .Spec.EnableVerboseLog }}
         - --verbose
         {{ end }}
         env:
diff --git a/k8s/pkg/templates/csidriver/deployment.yaml b/k8s/pkg/templates/csidriver/deployment.yaml
index 55d946fa..f9c14cb3 100644
--- a/k8s/pkg/templates/csidriver/deployment.yaml
+++ b/k8s/pkg/templates/csidriver/deployment.yaml
@@ -26,7 +26,7 @@ spec:
           - --endpoint=$(CSI_ENDPOINT)
           - --nodeid=$(KUBE_NODE_NAME)
           - --state-file-path=/csi/state
-          {{ if .Spec.EnableDebugLog }}
+          {{ if .Spec.EnableVerboseLog }}
           - --verbose
           {{ end }}
           env:

From 4e780397807e3e728f7431b2182c41ea0dfa55f7 Mon Sep 17 00:00:00 2001
From: Ye Cao <caoye.cao@alibaba-inc.com>
Date: Mon, 25 Sep 2023 09:51:13 +0800
Subject: [PATCH 19/22] Map multiple vineyard objects to one volume while
 specifying the common prefix in the kubeflow benchmark (#1578)

* Map multiple vineyard objects to one volume while specifying the common prefix.
* Update the kubeflow benchmark result and the related tutorial.

Fixes #1568

Signed-off-by: Ye Cao <caoye.cao@alibaba-inc.com>
---
 ...g-in-kubeflow-with-vineyard-csi-driver.rst |  33 ++--
 k8s/examples/vineyard-csidriver/Dockerfile    |   2 -
 .../pipeline-with-vineyard.py                 |  32 +---
 .../pipeline-with-vineyard.yaml               | 165 +++---------------
 .../preprocess/preprocess.py                  |   8 +-
 k8s/examples/vineyard-csidriver/test/test.py  |   4 +-
 .../vineyard-csidriver/train/train.py         |   6 +-
 7 files changed, 60 insertions(+), 190 deletions(-)

diff --git a/docs/tutorials/kubernetes/efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst b/docs/tutorials/kubernetes/efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst
index 675906b5..f1fb12c7 100644
--- a/docs/tutorials/kubernetes/efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst
+++ b/docs/tutorials/kubernetes/efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst
@@ -13,6 +13,7 @@ Prerequisites
 - A kubernetes cluster with version >= 1.25.10. If you don't have one by hand, you can refer to the guide `Initialize Kubernetes Cluster`_ to create one.
 - Install the `Vineyardctl`_ by following the official guide.
 - Install the `Argo Workflow CLI`_ by following the official guide.
+- Install the kfp package with version < 2.0.0.
 
 Deploy the Vineyard Cluster
 ===========================
@@ -107,6 +108,14 @@ in the first step will be mapped to a volume. Notice, the volume used in any tas
 explicitly mounted to the corresponding path in the source code, and the storageclass_name 
 format of each VolumeOp is ``{vineyard-deployment-namespace}.{vineyard-deployment-name}.csi``.
 
+There are two ways to add the ``vineyard object`` VolumeOp to the pipeline's dependencies:
+
+- Each path in the source code is mapped to a volume, and each volume is mounted to the actual path
+in the source code. The benefit is that the source path does not need to be modified.
+- Create a volume for the paths with the same prefix in the source code. You can add the prefix ``/vineyard``
+for the paths in the source code, and mount a volume to the path ``/vineyard``. In this way, you can
+only create one volume for multiple paths/vineyard objects.
+
 You may get some insights from the modified pipeline ``pipeline-with-vineyard.py``. Then, we need to
 compile the pipeline to an argo-workflow yaml. To be compatible with benchmark test, we update the
 generated ``pipeline.yaml`` and ``pipeline-with-vineyard.yaml``.
@@ -189,11 +198,11 @@ Argo workflow duration
 +------------+------------------+---------------+
 | data scale | without vineyard | with vineyard |
 +============+==================+===============+
-| 8500 Mi    | 186s             | 169s          |
+| 8500 Mi    | 189s             | 164s          |
 +------------+------------------+---------------+
-| 12000 Mi   | 250s             | 203s          |
+| 12000 Mi   | 234s             | 199s          |
 +------------+------------------+---------------+
-| 15000 Mi   | 332s             | 286s          |
+| 15000 Mi   | 298s             | 252s          |
 +------------+------------------+---------------+
 
 
@@ -211,11 +220,11 @@ Actual execution time
 +------------+------------------+---------------+
 | data scale | without vineyard | with vineyard |
 +============+==================+===============+
-| 8500 Mi    | 139.3s           | 92.3s         |
+| 8500 Mi    | 142.2s           | 94.3s         |
 +------------+------------------+---------------+
-| 12000 Mi   | 204.3s           | 131.1s        |
+| 12000 Mi   | 191.2s           | 123.1s        |
 +------------+------------------+---------------+
-| 15000 Mi   | 289.3s           | 209.7s        |
+| 15000 Mi   | 253.5s           | 181.4s        |
 +------------+------------------+---------------+
 
 
@@ -229,11 +238,11 @@ Writing time
 +------------+------------------+---------------+
 | data scale | without vineyard | with vineyard |
 +============+==================+===============+
-| 8500 Mi    | 21s              | 5.4s          |
+| 8500 Mi    | 21.6s            | 5.5s          |
 +------------+------------------+---------------+
-| 12000 Mi   | 26s              | 7s            |
+| 12000 Mi   | 26.6s            | 6.8s          |
 +------------+------------------+---------------+
-| 15000 Mi   | 32.2s            | 9.4s          |
+| 15000 Mi   | 32.7s            | 9.2s          |
 +------------+------------------+---------------+
 
 
@@ -252,11 +261,11 @@ We delete the time of init data loading, and the results are as follows:
 +------------+------------------+---------------+
 | data scale | without vineyard | with vineyard |
 +============+==================+===============+
-| 8500 Mi    | 36.7s            | 0.02s         |
+| 8500 Mi    | 37.3s            | 0.04s         |
 +------------+------------------+---------------+
-| 12000 Mi   | 45.7s            | 0.02s         |
+| 12000 Mi   | 49.5s            | 0.04s         |
 +------------+------------------+---------------+
-| 15000 Mi   | 128.6s           | 0.04s         |
+| 15000 Mi   | 61.7s            | 0.04s         |
 +------------+------------------+---------------+
 
 Based on the above results, we can find that the read time of vineyard is
diff --git a/k8s/examples/vineyard-csidriver/Dockerfile b/k8s/examples/vineyard-csidriver/Dockerfile
index 728442cc..612e2d22 100644
--- a/k8s/examples/vineyard-csidriver/Dockerfile
+++ b/k8s/examples/vineyard-csidriver/Dockerfile
@@ -8,5 +8,3 @@ ARG APP
 ENV APP ${APP}
 
 COPY ${APP} /${APP}
-
-
diff --git a/k8s/examples/vineyard-csidriver/pipeline-with-vineyard.py b/k8s/examples/vineyard-csidriver/pipeline-with-vineyard.py
index 6d41d30c..ef65c5bf 100644
--- a/k8s/examples/vineyard-csidriver/pipeline-with-vineyard.py
+++ b/k8s/examples/vineyard-csidriver/pipeline-with-vineyard.py
@@ -3,29 +3,12 @@
 def PreProcess():
     #############################################################
     # user need to add the volume Op for vineyard object manully
-    v1op = dsl.VolumeOp(name="vineyard-object1",
-                    resource_name="vineyard-object1-pvc", size="1Mi",
+    vop = dsl.VolumeOp(name="vineyard-objects",
+                    resource_name="vineyard-objects-pvc", size="1Mi",
                     storage_class="vineyard-system.vineyardd-sample.csi",
                     modes=dsl.VOLUME_MODE_RWM,
                     set_owner_reference=True)
 
-    v2op = dsl.VolumeOp(name="vineyard-object2",
-                resource_name="vineyard-object2-pvc", size="1Mi",
-                storage_class="vineyard-system.vineyardd-sample.csi",
-                modes=dsl.VOLUME_MODE_RWM,
-                set_owner_reference=True)
-
-    v3op = dsl.VolumeOp(name="vineyard-object3",
-            resource_name="vineyard-object3-pvc", size="1Mi",
-            storage_class="vineyard-system.vineyardd-sample.csi",
-            modes=dsl.VOLUME_MODE_RWM,
-            set_owner_reference=True)
-
-    v4op = dsl.VolumeOp(name="vineyard-object4",
-                    resource_name="vineyard-object4-pvc", size="1Mi",
-                    storage_class="vineyard-system.vineyardd-sample.csi",
-                    modes=dsl.VOLUME_MODE_RWM,
-                    set_owner_reference=True)
     #############################################################
 
     return dsl.ContainerOp(
@@ -34,10 +17,7 @@ def PreProcess():
         container_kwargs={'image_pull_policy':"IfNotPresent"},
         pvolumes={
             "/data": dsl.PipelineVolume(pvc="benchmark-data"),
-            '/data/x_train.pkl': v1op.volume,
-            '/data/y_train.pkl': v2op.volume,
-            '/data/x_test.pkl': v3op.volume,
-            '/data/y_test.pkl': v4op.volume
+            "/vineyard/data": vop.volume
         },
         command = ['python3', 'preprocess.py']
     )
@@ -49,8 +29,7 @@ def Train(comp1):
         container_kwargs={'image_pull_policy':"IfNotPresent"},
         pvolumes={
             "/data": comp1.pvolumes['/data'],
-            '/data/x_train.pkl': comp1.pvolumes['/data/x_train.pkl'],
-            '/data/y_train.pkl': comp1.pvolumes['/data/y_train.pkl']
+            "/vineyard/data": comp1.pvolumes['/vineyard/data'],
         },
         command = ['python3', 'train.py'],
     )
@@ -62,8 +41,7 @@ def Test(comp1, comp2):
         container_kwargs={'image_pull_policy':"IfNotPresent"},
         pvolumes={
             "/data": comp2.pvolumes['/data'],
-            '/data/x_test.pkl': comp1.pvolumes['/data/x_test.pkl'],
-            '/data/y_test.pkl': comp1.pvolumes['/data/y_test.pkl']
+            "/vineyard/data": comp1.pvolumes['/vineyard/data']
         },
         command = ['python3', 'test.py']
     )
diff --git a/k8s/examples/vineyard-csidriver/pipeline-with-vineyard.yaml b/k8s/examples/vineyard-csidriver/pipeline-with-vineyard.yaml
index 1b943fc6..00da241d 100644
--- a/k8s/examples/vineyard-csidriver/pipeline-with-vineyard.yaml
+++ b/k8s/examples/vineyard-csidriver/pipeline-with-vineyard.yaml
@@ -2,7 +2,7 @@ apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
   generateName: machine-learning-pipeline-
-  annotations: {pipelines.kubeflow.org/kfp_sdk_version: 1.8.0, pipelines.kubeflow.org/pipeline_compilation_time: '2023-09-13T13:58:28.716735',
+  annotations: {pipelines.kubeflow.org/kfp_sdk_version: 1.8.0, pipelines.kubeflow.org/pipeline_compilation_time: '2023-09-21T19:11:42.500362',
     pipelines.kubeflow.org/pipeline_spec: '{"description": "An example pipeline that
       trains and logs a regression model.", "name": "Machine learning Pipeline"}'}
   labels: {pipelines.kubeflow.org/kfp_sdk_version: 1.8.0}
@@ -14,31 +14,23 @@ spec:
       tasks:
       - name: preprocess-data
         template: preprocess-data
-        dependencies: [vineyard-object1, vineyard-object2, vineyard-object3, vineyard-object4]
+        dependencies: [vineyard-objects]
         arguments:
           parameters:
-          - {name: vineyard-object1-name, value: '{{tasks.vineyard-object1.outputs.parameters.vineyard-object1-name}}'}
-          - {name: vineyard-object2-name, value: '{{tasks.vineyard-object2.outputs.parameters.vineyard-object2-name}}'}
-          - {name: vineyard-object3-name, value: '{{tasks.vineyard-object3.outputs.parameters.vineyard-object3-name}}'}
-          - {name: vineyard-object4-name, value: '{{tasks.vineyard-object4.outputs.parameters.vineyard-object4-name}}'}
+          - {name: vineyard-objects-name, value: '{{tasks.vineyard-objects.outputs.parameters.vineyard-objects-name}}'}
       - name: test-data
         template: test-data
-        dependencies: [preprocess-data, train-data, vineyard-object3, vineyard-object4]
+        dependencies: [preprocess-data, train-data, vineyard-objects]
         arguments:
           parameters:
-          - {name: vineyard-object3-name, value: '{{tasks.vineyard-object3.outputs.parameters.vineyard-object3-name}}'}
-          - {name: vineyard-object4-name, value: '{{tasks.vineyard-object4.outputs.parameters.vineyard-object4-name}}'}
+          - {name: vineyard-objects-name, value: '{{tasks.vineyard-objects.outputs.parameters.vineyard-objects-name}}'}
       - name: train-data
         template: train-data
-        dependencies: [preprocess-data, vineyard-object1, vineyard-object2]
+        dependencies: [preprocess-data, vineyard-objects]
         arguments:
           parameters:
-          - {name: vineyard-object1-name, value: '{{tasks.vineyard-object1.outputs.parameters.vineyard-object1-name}}'}
-          - {name: vineyard-object2-name, value: '{{tasks.vineyard-object2.outputs.parameters.vineyard-object2-name}}'}
-      - {name: vineyard-object1, template: vineyard-object1}
-      - {name: vineyard-object2, template: vineyard-object2}
-      - {name: vineyard-object3, template: vineyard-object3}
-      - {name: vineyard-object4, template: vineyard-object4}
+          - {name: vineyard-objects-name, value: '{{tasks.vineyard-objects.outputs.parameters.vineyard-objects-name}}'}
+      - {name: vineyard-objects, template: vineyard-objects}
   - name: preprocess-data
     container:
       command: [python3, preprocess.py]
@@ -53,16 +45,10 @@ spec:
         value: "{{workflow.parameters.data_multiplier}}"
       volumeMounts:
       - {mountPath: /data, name: pvolume-d9c6725a1237b14c08a2567cb12c489bec539873deeddba7d87f5b4}
-      - {mountPath: /data/x_train.pkl, name: vineyard-object1}
-      - {mountPath: /data/y_train.pkl, name: vineyard-object2}
-      - {mountPath: /data/x_test.pkl, name: vineyard-object3}
-      - {mountPath: /data/y_test.pkl, name: vineyard-object4}
+      - {mountPath: /vineyard/data, name: vineyard-objects}
     inputs:
       parameters:
-      - {name: vineyard-object1-name}
-      - {name: vineyard-object2-name}
-      - {name: vineyard-object3-name}
-      - {name: vineyard-object4-name}
+      - {name: vineyard-objects-name}
     metadata:
       labels:
         pipelines.kubeflow.org/kfp_sdk_version: 1.8.0
@@ -71,14 +57,8 @@ spec:
     volumes:
     - name: pvolume-d9c6725a1237b14c08a2567cb12c489bec539873deeddba7d87f5b4
       persistentVolumeClaim: {claimName: benchmark-data}
-    - name: vineyard-object1
-      persistentVolumeClaim: {claimName: '{{inputs.parameters.vineyard-object1-name}}'}
-    - name: vineyard-object2
-      persistentVolumeClaim: {claimName: '{{inputs.parameters.vineyard-object2-name}}'}
-    - name: vineyard-object3
-      persistentVolumeClaim: {claimName: '{{inputs.parameters.vineyard-object3-name}}'}
-    - name: vineyard-object4
-      persistentVolumeClaim: {claimName: '{{inputs.parameters.vineyard-object4-name}}'}
+    - name: vineyard-objects
+      persistentVolumeClaim: {claimName: '{{inputs.parameters.vineyard-objects-name}}'}
   - name: test-data
     container:
       command: [python3, test.py]
@@ -93,12 +73,10 @@ spec:
         value: "{{workflow.parameters.data_multiplier}}"
       volumeMounts:
       - {mountPath: /data, name: pvolume-d9c6725a1237b14c08a2567cb12c489bec539873deeddba7d87f5b4}
-      - {mountPath: /data/x_test.pkl, name: vineyard-object3}
-      - {mountPath: /data/y_test.pkl, name: vineyard-object4}
+      - {mountPath: /vineyard/data, name: vineyard-objects}
     inputs:
       parameters:
-      - {name: vineyard-object3-name}
-      - {name: vineyard-object4-name}
+      - {name: vineyard-objects-name}
     metadata:
       labels:
         pipelines.kubeflow.org/kfp_sdk_version: 1.8.0
@@ -107,10 +85,8 @@ spec:
     volumes:
     - name: pvolume-d9c6725a1237b14c08a2567cb12c489bec539873deeddba7d87f5b4
       persistentVolumeClaim: {claimName: benchmark-data}
-    - name: vineyard-object3
-      persistentVolumeClaim: {claimName: '{{inputs.parameters.vineyard-object3-name}}'}
-    - name: vineyard-object4
-      persistentVolumeClaim: {claimName: '{{inputs.parameters.vineyard-object4-name}}'}
+    - name: vineyard-objects
+      persistentVolumeClaim: {claimName: '{{inputs.parameters.vineyard-objects-name}}'}
   - name: train-data
     container:
       command: [python3, train.py]
@@ -125,12 +101,10 @@ spec:
         value: "{{workflow.parameters.data_multiplier}}"
       volumeMounts:
       - {mountPath: /data, name: pvolume-d9c6725a1237b14c08a2567cb12c489bec539873deeddba7d87f5b4}
-      - {mountPath: /data/x_train.pkl, name: vineyard-object1}
-      - {mountPath: /data/y_train.pkl, name: vineyard-object2}
+      - {mountPath: /vineyard/data, name: vineyard-objects}
     inputs:
       parameters:
-      - {name: vineyard-object1-name}
-      - {name: vineyard-object2-name}
+      - {name: vineyard-objects-name}
     metadata:
       labels:
         pipelines.kubeflow.org/kfp_sdk_version: 1.8.0
@@ -139,11 +113,9 @@ spec:
     volumes:
     - name: pvolume-d9c6725a1237b14c08a2567cb12c489bec539873deeddba7d87f5b4
       persistentVolumeClaim: {claimName: benchmark-data}
-    - name: vineyard-object1
-      persistentVolumeClaim: {claimName: '{{inputs.parameters.vineyard-object1-name}}'}
-    - name: vineyard-object2
-      persistentVolumeClaim: {claimName: '{{inputs.parameters.vineyard-object2-name}}'}
-  - name: vineyard-object1
+    - name: vineyard-objects
+      persistentVolumeClaim: {claimName: '{{inputs.parameters.vineyard-objects-name}}'}
+  - name: vineyard-objects
     resource:
       action: create
       setOwnerReference: true
@@ -151,7 +123,7 @@ spec:
         apiVersion: v1
         kind: PersistentVolumeClaim
         metadata:
-          name: '{{workflow.name}}-vineyard-object1-pvc'
+          name: '{{workflow.name}}-vineyard-objects-pvc'
         spec:
           accessModes:
           - ReadWriteMany
@@ -161,98 +133,11 @@ spec:
           storageClassName: vineyard-system.vineyardd-sample.csi
     outputs:
       parameters:
-      - name: vineyard-object1-manifest
+      - name: vineyard-objects-manifest
         valueFrom: {jsonPath: '{}'}
-      - name: vineyard-object1-name
+      - name: vineyard-objects-name
         valueFrom: {jsonPath: '{.metadata.name}'}
-      - name: vineyard-object1-size
-        valueFrom: {jsonPath: '{.status.capacity.storage}'}
-    metadata:
-      labels:
-        pipelines.kubeflow.org/kfp_sdk_version: 1.8.0
-        pipelines.kubeflow.org/pipeline-sdk-type: kfp
-        pipelines.kubeflow.org/enable_caching: "true"
-  - name: vineyard-object2
-    resource:
-      action: create
-      setOwnerReference: true
-      manifest: |
-        apiVersion: v1
-        kind: PersistentVolumeClaim
-        metadata:
-          name: '{{workflow.name}}-vineyard-object2-pvc'
-        spec:
-          accessModes:
-          - ReadWriteMany
-          resources:
-            requests:
-              storage: 1Mi
-          storageClassName: vineyard-system.vineyardd-sample.csi
-    outputs:
-      parameters:
-      - name: vineyard-object2-manifest
-        valueFrom: {jsonPath: '{}'}
-      - name: vineyard-object2-name
-        valueFrom: {jsonPath: '{.metadata.name}'}
-      - name: vineyard-object2-size
-        valueFrom: {jsonPath: '{.status.capacity.storage}'}
-    metadata:
-      labels:
-        pipelines.kubeflow.org/kfp_sdk_version: 1.8.0
-        pipelines.kubeflow.org/pipeline-sdk-type: kfp
-        pipelines.kubeflow.org/enable_caching: "true"
-  - name: vineyard-object3
-    resource:
-      action: create
-      setOwnerReference: true
-      manifest: |
-        apiVersion: v1
-        kind: PersistentVolumeClaim
-        metadata:
-          name: '{{workflow.name}}-vineyard-object3-pvc'
-        spec:
-          accessModes:
-          - ReadWriteMany
-          resources:
-            requests:
-              storage: 1Mi
-          storageClassName: vineyard-system.vineyardd-sample.csi
-    outputs:
-      parameters:
-      - name: vineyard-object3-manifest
-        valueFrom: {jsonPath: '{}'}
-      - name: vineyard-object3-name
-        valueFrom: {jsonPath: '{.metadata.name}'}
-      - name: vineyard-object3-size
-        valueFrom: {jsonPath: '{.status.capacity.storage}'}
-    metadata:
-      labels:
-        pipelines.kubeflow.org/kfp_sdk_version: 1.8.0
-        pipelines.kubeflow.org/pipeline-sdk-type: kfp
-        pipelines.kubeflow.org/enable_caching: "true"
-  - name: vineyard-object4
-    resource:
-      action: create
-      setOwnerReference: true
-      manifest: |
-        apiVersion: v1
-        kind: PersistentVolumeClaim
-        metadata:
-          name: '{{workflow.name}}-vineyard-object4-pvc'
-        spec:
-          accessModes:
-          - ReadWriteMany
-          resources:
-            requests:
-              storage: 1Mi
-          storageClassName: vineyard-system.vineyardd-sample.csi
-    outputs:
-      parameters:
-      - name: vineyard-object4-manifest
-        valueFrom: {jsonPath: '{}'}
-      - name: vineyard-object4-name
-        valueFrom: {jsonPath: '{.metadata.name}'}
-      - name: vineyard-object4-size
+      - name: vineyard-objects-size
         valueFrom: {jsonPath: '{.status.capacity.storage}'}
     metadata:
       labels:
diff --git a/k8s/examples/vineyard-csidriver/preprocess/preprocess.py b/k8s/examples/vineyard-csidriver/preprocess/preprocess.py
index 64237f0f..e28228df 100644
--- a/k8s/examples/vineyard-csidriver/preprocess/preprocess.py
+++ b/k8s/examples/vineyard-csidriver/preprocess/preprocess.py
@@ -61,10 +61,10 @@ def preprocess_data():
     st = time.time()
     with_vineyard = os.environ.get('WITH_VINEYARD', False)
     if with_vineyard:
-        vineyard.csi.write(X_train, "/data/x_train.pkl")
-        vineyard.csi.write(X_test, "/data/x_test.pkl")
-        vineyard.csi.write(y_train, "/data/y_train.pkl")
-        vineyard.csi.write(y_test, "/data/y_test.pkl")
+        vineyard.csi.write(X_train, "/vineyard/data/x_train.pkl")
+        vineyard.csi.write(X_test, "/vineyard/data/x_test.pkl")
+        vineyard.csi.write(y_train, "/vineyard/data/y_train.pkl")
+        vineyard.csi.write(y_test, "/vineyard/data/y_test.pkl")
     else:
         X_train.to_pickle('/data/x_train.pkl')
         X_test.to_pickle('/data/x_test.pkl')
diff --git a/k8s/examples/vineyard-csidriver/test/test.py b/k8s/examples/vineyard-csidriver/test/test.py
index 97d5d0da..57685ebb 100644
--- a/k8s/examples/vineyard-csidriver/test/test.py
+++ b/k8s/examples/vineyard-csidriver/test/test.py
@@ -12,8 +12,8 @@ def test_model():
     with_vineyard = os.environ.get('WITH_VINEYARD', False)
     st = time.time()
     if with_vineyard:
-        x_test_data = vineyard.csi.read("/data/x_test.pkl")
-        y_test_data = vineyard.csi.read("/data/y_test.pkl")
+        x_test_data = vineyard.csi.read("/vineyard/data/x_test.pkl")
+        y_test_data = vineyard.csi.read("/vineyard/data/y_test.pkl")
     else:
         x_test_data = pd.read_pickle("/data/x_test.pkl")
         y_test_data = pd.read_pickle("/data/y_test.pkl")
diff --git a/k8s/examples/vineyard-csidriver/train/train.py b/k8s/examples/vineyard-csidriver/train/train.py
index 93a71c77..724cb6f9 100644
--- a/k8s/examples/vineyard-csidriver/train/train.py
+++ b/k8s/examples/vineyard-csidriver/train/train.py
@@ -9,12 +9,12 @@
 
 
 def train_model():
-    os.system('echo 3 > /proc/sys/vm/drop_caches')
+    os.system('sync; echo 3 > /proc/sys/vm/drop_caches')
     st = time.time()
     with_vineyard = os.environ.get('WITH_VINEYARD', False)
     if with_vineyard:
-        x_train_data = vineyard.csi.read("/data/x_train.pkl")
-        y_train_data = vineyard.csi.read("/data/y_train.pkl")
+        x_train_data = vineyard.csi.read("/vineyard/data/x_train.pkl")
+        y_train_data = vineyard.csi.read("/vineyard/data/y_train.pkl")
     else:
         x_train_data = pd.read_pickle("/data/x_train.pkl")
         y_train_data = pd.read_pickle("/data/y_train.pkl")

From 77714603558578c8d000e7152c16291a70486197 Mon Sep 17 00:00:00 2001
From: Tao He <linzhu.ht@alibaba-inc.com>
Date: Mon, 25 Sep 2023 13:51:32 +0800
Subject: [PATCH 20/22] Add docs for gpu memory sharing in vineyard (#1581)

Signed-off-by: Tao He <linzhu.ht@alibaba-inc.com>
---
 docs/tutorials/data-processing.rst            |  11 +
 .../data-processing/gpu-memory-sharing.rst    | 227 ++++++++++++++++++
 src/client/client.cc                          |  14 ++
 src/client/client.h                           |  13 +-
 test/gpumalloc_test.cu                        |  11 +-
 5 files changed, 271 insertions(+), 5 deletions(-)
 create mode 100644 docs/tutorials/data-processing/gpu-memory-sharing.rst

diff --git a/docs/tutorials/data-processing.rst b/docs/tutorials/data-processing.rst
index 700226e6..2d4ecc1d 100644
--- a/docs/tutorials/data-processing.rst
+++ b/docs/tutorials/data-processing.rst
@@ -11,6 +11,7 @@ Data processing
    ./data-processing/python-sharedmemory.rst
    ./data-processing/distributed-learning.rst
    ./data-processing/accelerate-data-sharing-in-kedro.rst
+   ./data-processing/gpu-memory-sharing.rst
 
 In these comprehensive case studies, we demonstrate how to seamlessly integrate vineyard's
 capabilities with existing data-intensive tasks. By incorporating vineyard into complex
@@ -57,3 +58,13 @@ improvements in both performance and ease of use.
    Vineyard serves as the :code:`DataSet` backend for Kedro pipelines, enabling
    efficient data sharing between tasks without intrusive code modification, even
    when the pipeline is deployed to Kubernetes.
+
+   ---
+
+   .. link-button:: ./data-processing/gpu-memory-sharing
+      :type: ref
+      :text: GPU Memory Sharing
+      :classes: btn-block stretched-link
+   ^^^^^^^^^^^^
+   Vineyard supports sharing GPU memory in zero-copy manner, enabling efficient data sharing
+   between GPU-accelerated tasks.
diff --git a/docs/tutorials/data-processing/gpu-memory-sharing.rst b/docs/tutorials/data-processing/gpu-memory-sharing.rst
new file mode 100644
index 00000000..8d3d9c95
--- /dev/null
+++ b/docs/tutorials/data-processing/gpu-memory-sharing.rst
@@ -0,0 +1,227 @@
+.. _gpu-memory-sharing:
+
+Sharing GPU Memory
+------------------
+
+Vineyard supports sharing both CPU memory and GPU memory between different
+processes and different compute engines. The sharing of GPU memory is archived
+by using the `CUDA IPC mechanism <https://docs.nvidia.com/cuda/cuda-runtime-api/structcudaIpcMemHandle__t.html>`_
+and provides a flexible unified memory interfaces.
+
+CUDA IPC and Unified Memory
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The CUDA IPC memory handle allows GPU memory to be shared between different
+processes via IPC. In vineyard, the GPU memory is allocated by the vineyardd
+instance when :code:`CreateGPUBuffer()`, then an IPC handle is transferred to the
+client process and the GPU memory can be accessed by the client process after
+calling :code:`cudaIpcOpenMemHandle()`. For readers, the GPU memory can be accessed
+like a normal CPU shared memory object with :code:`GetGPUBuffers()`.
+
+Like `CUDA unified memory <https://developer.nvidia.com/blog/unified-memory-cuda-beginners/>`_,
+vineyard's provides a unified memory interface which can be adapted to different
+kinds of implementation (GPU, PPU, etc.) as the abstraction to share GPU memory
+between different processes, as well as sharing memory between the host and
+device.
+
+The unified memory abstractions provides the following utilities:
+
+- Accessing the memory from GPU or CPU using an unified abstraction: the
+  :code:`GPUData()` and :code:`CPUData()`;
+- Synchronizing the memory between GPU and CPU: the :code:`syncFromCPU()` and
+  :code:`syncFromGPU()`.
+
+Example
+-------
+
+.. note::
+
+    The GPU shared memory is still under development and the APIs may change in
+    the future.
+
+- Creating a GPU buffer:
+
+  .. code:: c++
+
+      ObjectID object_id;
+      Payload object;
+      std::shared_ptr<GPUUnifiedAddress> gua = nullptr;
+      RETURN_ON_ERROR(client.CreateGPUBuffer(data_size(), object_id, object, gua));
+
+- Write data to the GPU buffer:
+
+  .. code:: c++
+
+      void* gpu_ptr = nullptr;
+      RETURN_ON_ERROR(gua->GPUData(&gpu_ptr));
+      cudaMemcpy(gpu_ptr, data, data_size(), cudaMemcpyHostToDevice);
+
+  or, copy to CPU memory first and then synchronize to GPU memory (like CUDA unified memory):
+
+  .. code:: c++
+
+      void* cpu_ptr = nullptr;
+      RETURN_ON_ERROR(gua->CPUData(&cpu_ptr));
+      memcpy(cpu_ptr, data, data_size());
+      RETURN_ON_ERROR(gua->syncFromCPU());
+
+- Accessing the GPU buffer from another process:
+
+  .. code:: c++
+
+      ObjectID object_id = ...;
+      std::shared_ptr<GPUUnifiedAddress> gua = nullptr;
+      RETURN_ON_ERROR(client.GetGPUBuffer(object_id, true, gua));
+
+      void* gpu_ptr = nullptr;
+      RETURN_ON_ERROR(gua->GPUData(&gpu_ptr));
+
+- Accessing the shared GPU buffer from CPU:
+
+  .. code:: c++
+
+      ObjectID object_id = ...;
+      std::shared_ptr<GPUUnifiedAddress> gua = nullptr;
+      RETURN_ON_ERROR(client.GetGPUBuffer(object_id, true, gua));
+
+      void* cpu_ptr = nullptr;
+      RETURN_ON_ERROR(gua->CPUData(&cpu_ptr));
+      RETURN_ON_ERROR(gua->syncFromGPU());
+
+- Freeing the shared GPU buffer:
+
+  .. code:: c++
+
+      ObjectID object_id = ...;
+      RETURN_ON_ERROR(client.DelData(object_id));
+
+:code:`UnifiedMemory` APIs
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The complete :code:`UnifiedMemory` APIs are defined as:
+
+.. code:: c++
+
+    class GPUUnifiedAddress {
+      /**
+      * @brief get the cpu memry address
+      *
+      * @param ptr the return cpu data address
+      * @return GUAError_t the error type
+      */
+      GUAError_t CPUData(void** ptr);
+
+      /**
+      * @brief get the gpu memory address
+      *
+      * @param ptr the return gpu data address
+      * @return GUAError_t the error type
+      */
+      GUAError_t GPUData(void** ptr);
+
+      /**
+      * @brief sync data from GPU related to this gua
+      *
+      * @return GUAError_t the error type
+      */
+      GUAError_t syncFromCPU();
+
+      /**
+      * @brief sync data from CPU related to this gua
+      *
+      * @return GUAError_t the error type
+      */
+      GUAError_t syncFromGPU();
+
+      /**
+      * @brief  Malloc memory related to this gua if needed.
+      *
+      * @param size the memory size to be allocated
+      * @param ptr the memory address on cpu or GPU
+      * @param is_GPU allocate on GPU
+      * @return GUAError_t the error type
+      */
+
+      GUAError_t ManagedMalloc(size_t size, void** ptr, bool is_GPU = false);
+      /**
+      * @brief Free the memory
+      *
+      */
+      void ManagedFree();
+
+      /**
+      * @brief GUA to json
+      *
+      */
+      void GUAToJSON();
+
+      /**
+      * @brief Get the Ipc Handle object
+      *
+      * @param handle the returned handle
+      * @return GUAError_t the error type
+      */
+
+      GUAError_t getIpcHandle(cudaIpcMemHandle_t& handle);
+      /**
+      * @brief Set the IpcHandle of this GUA
+      *
+      * @param handle
+      */
+      void setIpcHandle(cudaIpcMemHandle_t handle);
+
+      /**
+      * @brief Get the IpcHandle of this GUA as vector
+      *
+      * @return std::vector<int64_t>
+      */
+      std::vector<int64_t> getIpcHandleVec();
+
+      /**
+      * @brief Set the IpcHandle vector of this GUA
+      *
+      * @param handle_vec
+      */
+      void setIpcHandleVec(std::vector<int64_t> handle_vec);
+
+      /**
+      * @brief Set the GPU Mem Ptr object
+      *
+      * @param ptr
+      */
+      void setGPUMemPtr(void* ptr);
+
+      /**
+      * @brief return the GPU memory pointer
+      *
+      * @return void* the GPU-side memory address
+      */
+      void* getGPUMemPtr();
+      /**
+      * @brief Set the Cpu Mem Ptr object
+      *
+      * @param ptr
+      */
+      void setCPUMemPtr(void* ptr);
+
+      /**
+      * @brief Get the Cpu Mem Ptr object
+      *
+      * @return void*
+      */
+      void* getCPUMemPtr();
+
+      /**
+      * @brief Get the Size object
+      *
+      * @return int64_t
+      */
+      int64_t getSize();
+
+      /**
+      * @brief Set the Size object
+      *
+      * @param data_size
+      */
+      void setSize(int64_t data_size);
+    };
diff --git a/src/client/client.cc b/src/client/client.cc
index 2044830a..8a8d40e7 100644
--- a/src/client/client.cc
+++ b/src/client/client.cc
@@ -666,6 +666,20 @@ Status Client::GetGPUBuffers(const std::set<ObjectID>& ids, const bool unsafe,
   return Status::OK();
 }
 
+Status Client::GetGPUBuffer(const ObjectID id, const bool unsafe,
+                            GPUUnifiedAddress& gua) {
+  std::set<ObjectID> ids;
+  ids.insert(id);
+  std::map<ObjectID, GPUUnifiedAddress> guas;
+  RETURN_ON_ERROR(GetGPUBuffers(ids, unsafe, guas));
+  if (guas.empty()) {
+    return Status::ObjectNotExists("buffer not exists: " +
+                                   ObjectIDToString(id));
+  }
+  gua = guas.at(id);
+  return Status::OK();
+}
+
 Status Client::GetBuffer(const ObjectID id, std::shared_ptr<Buffer>& buffer) {
   std::map<ObjectID, std::shared_ptr<Buffer>> buffers;
   RETURN_ON_ERROR(GetBuffers({id}, buffers));
diff --git a/src/client/client.h b/src/client/client.h
index 22a30b36..09bfa679 100644
--- a/src/client/client.h
+++ b/src/client/client.h
@@ -790,13 +790,24 @@ class Client final : public BasicIPCClient,
    * @brief Get a set of blobs from vineyard server. See also `GetBuffer`.
    *
    * @param ids Object ids for the blobs to get.
-   * @param buffers: The result result cudaIpcMemhandles related to GPU blobs.
+   * @param GUAs: The result unified memory objects related to GPU blobs.
    *
    * @return Status that indicates whether the get action has succeeded.
    */
   Status GetGPUBuffers(const std::set<ObjectID>& ids, const bool unsafe,
                        std::map<ObjectID, GPUUnifiedAddress>& GUAs);
 
+  /**
+   * @brief Get a single GPU blob from vineyard server. See also `GetBuffer`.
+   *
+   * @param id Object id for the blob to get.
+   * @param buffer: The result unified memory object related to the GPU blob.
+   *
+   * @return Status that indicates whether the get action has succeeded.
+   */
+  Status GetGPUBuffer(const ObjectID id, const bool unsafe,
+                      GPUUnifiedAddress& gua);
+
  protected:
   /**
    * @brief Required by `UsageTracker`. When reference count reaches zero, send
diff --git a/test/gpumalloc_test.cu b/test/gpumalloc_test.cu
index 80e7cefa..4ead8b64 100644
--- a/test/gpumalloc_test.cu
+++ b/test/gpumalloc_test.cu
@@ -79,17 +79,20 @@ int main(int argc, char** argv) {
   std::string ipc_socket = std::string(argv[1]);
   //connection test
   Client client;
-  VINEYARD_CHECK_OK(client.BasicIPCClient::Open(ipc_socket, StoreType::kDefault));
-  LOG(INFO) << "Connected to IPCServer: type kGPU" << ipc_socket;
+  VINEYARD_CHECK_OK(client.Connect(ipc_socket));
+  LOG(INFO) << "Connected to IPCServer: " << ipc_socket;
+
   std::vector<std::string> data;
   data.emplace_back("Create Success.");
   data.emplace_back("hello world!");
+
   std::vector<ObjectID> oids(data.size());
   LOG(INFO) << "Create GPU Object tests\n"; 
-  create_gpu_objects(client, data, false, oids);
+  VINEYARD_CHECK_OK(create_gpu_objects(client, data, false, oids));
   LOG(INFO) << "Passed GPU create test...";
+
   LOG(INFO) << "Get GPU Object tests\n"; 
-  get_gpu_objects(client, data, oids, false);
+  VINEYARD_CHECK_OK(get_gpu_objects(client, data, oids, false));
   LOG(INFO) << "Passed GPU get test...";
   client.Disconnect();
 

From cd22161e7cc6ac1bc2a977517f64cd85edddcf21 Mon Sep 17 00:00:00 2001
From: Tao He <linzhu.ht@alibaba-inc.com>
Date: Mon, 25 Sep 2023 14:02:00 +0800
Subject: [PATCH 21/22] Fixes the markup

Signed-off-by: Tao He <linzhu.ht@alibaba-inc.com>
---
 docs/tutorials/data-processing/gpu-memory-sharing.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/tutorials/data-processing/gpu-memory-sharing.rst b/docs/tutorials/data-processing/gpu-memory-sharing.rst
index 8d3d9c95..5739c408 100644
--- a/docs/tutorials/data-processing/gpu-memory-sharing.rst
+++ b/docs/tutorials/data-processing/gpu-memory-sharing.rst
@@ -32,7 +32,7 @@ The unified memory abstractions provides the following utilities:
   :code:`syncFromGPU()`.
 
 Example
--------
+~~~~~~~
 
 .. note::
 

From a6fc10ab9039d5fef1b1e4a598a3e1905a49da3d Mon Sep 17 00:00:00 2001
From: Ye Cao <caoye.cao@alibaba-inc.com>
Date: Mon, 25 Sep 2023 14:04:23 +0800
Subject: [PATCH 22/22] Remove the outdated vineyard-cli doc and improve python
 API References. (#1580)

- Remove the outdated vineyard-cli doc.
- Add the `vineyard.csi.read/write` API and `python -m vineyard.ctl` API to the python API References.

Fixes #1579

Signed-off-by: Ye Cao <caoye.cao@alibaba-inc.com>
---
 .github/build-docs.sh                |   2 +-
 docs/notes/references.rst            |  15 --
 docs/notes/references/ctl.rst        | 285 ---------------------------
 docs/notes/references/python-api.rst |  18 ++
 4 files changed, 19 insertions(+), 301 deletions(-)
 delete mode 100644 docs/notes/references/ctl.rst

diff --git a/.github/build-docs.sh b/.github/build-docs.sh
index e4f00ed9..695faa8a 100755
--- a/.github/build-docs.sh
+++ b/.github/build-docs.sh
@@ -12,7 +12,7 @@ pip3 install -r requirements-setup.txt -r requirements.txt -r requirements-dev.t
 pip3 install black isort flake8
 
 # build vineyard_client_python
-mkdir build
+mkdir -p build
 pushd build
 cmake .. -DCMAKE_BUILD_TYPE=Debug \
          -DBUILD_SHARED_LIBS=ON \
diff --git a/docs/notes/references.rst b/docs/notes/references.rst
index 9e3305ca..9d3cc665 100644
--- a/docs/notes/references.rst
+++ b/docs/notes/references.rst
@@ -8,7 +8,6 @@ API Reference
 
    references/python-api.rst
    references/cpp-api.rst
-   references/ctl.rst
    Kubernetes CRDs <references/crds.md>
 
 Vineyard offers a comprehensive suite of SDKs, including Python and C++ versions.
@@ -34,20 +33,6 @@ For detailed API references, please explore the following pages:
    ^^^^^^^^^^^^
    API reference for vineyard C++ SDK.
 
-In addition, the command-line tool `vineyard-ctl` is available to facilitate interactions
-with a local :code:`vineyardd` instance, making inspection and debugging tasks more efficient.
-
-.. panels::
-   :header: text-center
-   :column: col-lg-12 p-2
-
-   .. link-button:: references/ctl
-      :type: ref
-      :text: Vineyard CTL
-      :classes: btn-block stretched-link
-   ^^^^^^^^^^^^
-   Reference for vineyard command line tools.
-
 All terms in the documentation site can use search from the following
 indexing page:
 
diff --git a/docs/notes/references/ctl.rst b/docs/notes/references/ctl.rst
deleted file mode 100644
index d275ebd9..00000000
--- a/docs/notes/references/ctl.rst
+++ /dev/null
@@ -1,285 +0,0 @@
-Vineyard Cli
-============
-
-**vineyard-ctl**: A command-line tool for **vineyard**.
-
-Connect to vineyard
--------------------
-
-+ Via command-line:
-
-  Options:
-
-  + :code:`ipc_socket`: Socket location of connected vineyard server.
-  + :code:`rpc_host`: RPC HOST of the connected vineyard server.
-  + :code:`rpc_port`: RPC PORT of the connected vineyard server.
-  + :code:`rpc_endpoint`: RPC endpoint of the connected vineyard server.
-
-  Example:
-
-  .. code:: shell
-
-      vineyard-ctl --ipc_socket /var/run/vineyard.sock
-
-+ Via vineyard configuration file:
-
-  This will pick IPC or RPC values from the vineyard configuration file or
-  environment variables.
-
-Supported Commands
-------------------
-
-+ :code:`ls`
-+ :code:`query`
-+ :code:`head`
-+ :code:`copy`
-+ :code:`del`
-+ :code:`stat`
-+ :code:`put`
-+ :code:`config`
-+ :code:`migrate`
-+ :code:`debug`
-+ :code:`start`
-
-.. note::
-
-    .. code:: shell
-
-        vineyard-ctl {command}
-
-:code:`ls`
-^^^^^^^^^^
-
-List vineyard objects.
-
-Options:
-
-+ :code:`pattern`: The pattern string that will be matched against the object’s typename.
-+ :code:`regex`: The pattern string will be considered as a regex expression.
-+ :code:`limit`: The limit to list.
-
-Example:
-
-.. code:: shell
-
-    vineyard-ctl ls --pattern * --regex --limit 8
-
-:code:`query`
-^^^^^^^^^^^^^
-
-Query a vineyard object.
-
-Options:
-
-+ :code:`object_id`: ID of the object to be fetched.
-+ :code:`meta`: Metadata of the object (**Simple** or **JSON**).
-+ :code:`metric`: Metric data of the object (**nbytes** or **signature** or **typename**).
-+ :code:`exists`: Check if the object exists or not.
-+ :code:`stdout`: Get object to stdout.
-+ :code:`output_file`: Get object to file.
-+ :code:`tree`: Get object lineage in tree-like style.
-+ :code:`memory_status`: Get the memory used by the vineyard object.
-+ :code:`detail`: Get detailed memory used by the vineyard object.
-
-Example:
-
-.. code:: shell
-
-    vineyard-ctl query --object_id 00002ec13bc81226 --meta json --metric typename
-
-:code:`head`
-^^^^^^^^^^^^
-
-Print first n(limit) lines of a vineyard object. Currently supported for a pandas dataframe only.
-
-Options:
-
-+ :code:`object_id`: ID of the object to be printed.
-+ :code:`limit`: Number of lines of the object to be printed.
-
-Example:
-
-.. code:: shell
-
-    vineyard-ctl head --object_id 00002ec13bc81226 --limit 3
-
-:code:`copy`
-^^^^^^^^^^^^
-
-Copy a vineyard object.
-
-Options:
-
-+ :code:`object_id`: ID of the object to be copied.
-+ :code:`shallow`: Get a shallow copy of the object.
-+ :code:`deep`: Get a deep copy of the object.
-
-Example:
-
-.. code:: shell
-
-    vineyard-ctl copy --object_id 00002ec13bc81226 --shallow
-
-:code:`del`
-^^^^^^^^^^^
-
-Delete a vineyard object.
-
-Options:
-
-+ :code:`object_id`: ID of the object to be deleted.
-+ :code:`regex_pattern`: Delete all the objects that match the regex pattern.
-+ :code:`force`: Recursively delete even if the member object is also referred by others.
-+ :code:`deep`: Deeply delete an object means we will deleting the members recursively.
-
-Example:
-
-.. code:: shell
-
-    vineyard-ctl del --object_id 00002ec13bc81226 --force
-
-:code:`stat`
-^^^^^^^^^^^^
-
-Get the status of connected vineyard server.
-
-Options:
-
-+ :code:`instance_id`: Instance ID of vineyardd that the client is connected to.
-+ :code:`deployment`: The deployment mode of the connected vineyardd cluster.
-+ :code:`memory_usage`: Memory usage (in bytes) of current vineyardd instance.
-+ :code:`memory_limit`: Memory limit (in bytes) of current vineyardd instance.
-+ :code:`deferred_requests`: Number of waiting requests of current vineyardd instance.
-+ :code:`ipc_connections`: Number of alive IPC connections on the current vineyardd instance.
-+ :code:`rpc_connections`: Number of alive RPC connections on the current vineyardd instance.
-
-Example:
-
-.. code:: shell
-
-    vineyard-ctl stat
-
-:code:`put`
-^^^^^^^^^^^
-
-Put a python value to vineyard.
-
-Options:
-
-+ :code:`value`: The python value you want to put to the vineyard server.
-+ :code:`file`: The file you want to put to the vineyard server as a pandas dataframe.
-+ :code:`sep`: Delimiter used in the file.
-+ :code:`delimiter`: Delimiter used in the file.
-+ :code:`header`: Row number to use as the column names.
-
-Example:
-
-.. code:: shell
-
-    vineyard-ctl put --file example_csv_file.csv --sep ,
-
-:code:`config`
-^^^^^^^^^^^^^^
-
-Edit configuration file.
-
-Options:
-
-+ :code:`ipc_socket_value`: The ipc_socket value to enter in the config file.
-+ :code:`rpc_host_value`: The rpc_host value to enter in the config file.
-+ :code:`rpc_port_value`: The rpc_port value to enter in the config file.
-+ :code:`rpc_endpoint_value`: The rpc_endpoint value to enter in the config file.
-
-Example:
-
-.. code:: shell
-
-    vineyard-ctl config --ipc_socket_value /var/run/vineyard.sock
-
-:code:`migrate`
-^^^^^^^^^^^^^^^
-
-Migrate a vineyard object.
-
-Options:
-
-+ :code:`ipc_socket_value`: The ipc_socket value for the second client.
-+ :code:`rpc_host_value`: The rpc_host value for the second client.
-+ :code:`rpc_port_value`: The rpc_port value for the second client.
-+ :code:`rpc_endpoint_value`: The rpc_endpoint value for the second client.
-+ :code:`object_id`: ID of the object to be migrated.
-+ :code:`local`: Migrate the vineyard object local to local.
-+ :code:`remote`: Migrate the vineyard object remote to local.
-
-Example:
-
-.. code:: shell
-
-    vineyard-ctl migrate --ipc_socket_value /tmp/vineyard.sock --object_id 00002ec13bc81226 --remote
-
-:code:`debug`
-^^^^^^^^^^^^^
-
-Issue a debug request.
-
-Options:
-
-+ :code:`payload`: The payload that will be sent to the debug handler.
-
-Example:
-
-.. code:: shell
-
-    vineyard-ctl debug --payload '{"instance_status":[], "memory_size":[]}'
-
-:code:`start`
-^^^^^^^^^^^^^
-
-Start vineyardd.
-
-Options:
-
-+ :code:`local`: start a local vineyard cluster.
-+ :code:`distributed`: start a local vineyard cluster in a distributed fashion.
-+ :code:`hosts`: A list of machines to launch vineyard server.
-+ :code:`etcd_endpoints`: Launching vineyard using specified etcd endpoints.
-  If not specified, vineyard will launch its own etcd instance.
-+ :code:`vineyardd_path`: Location of vineyard server program. If not specified,
-  vineyard will use its own bundled vineyardd binary.
-+ :code:`size`: The memory size limit for vineyard’s shared memory. The memory size
-  can be a plain integer or as a fixed-point number using one of these suffixes:
-  :code:`E`, :code:`P`, :code:`T`, :code:`G`, :code:`M`, :code:`K`. You can also
-  use the power-of-two equivalents: :code:`Ei`, :code:`Pi`, :code:`Ti`, :code:`Gi`,
-  :code:`Mi`, :code:`Ki`.
-+ :code:`socket`: The UNIX domain socket socket path that vineyard server will
-  bind and listen on. When the socket parameter is None, a random path under
-  temporary directory will be generated and used.
-+ :code:`rpc_socket_port`: The port that vineyard will use to privode RPC service.
-+ :code:`debug`: Whether to print debug logs.
-
-Example:
-
-.. code:: shell
-
-    vineyard-ctl start --local
-
-Autocomplete
-------------
-
-Autocomplete for vineyard-ctl is only supported for the bash shell currently.
-
-Follow the following steps to enable autocomplete for vineyard-ctl on your system:
-
-+ Install :code:`argcomplete` via :code:`pip3`: :code:`pip3 install argcomplete`.
-+ Copy the :code:`python/vineyard/cli.py` file to :code:`/usr/local/bin`.
-+ Add :code:`eval "$(register-python-argcomplete cli.py)"` to :code:`~/.bashrc`.
-+ Run :code:`source /etc/profile`.
-+ Run :code:`source ~/.bashrc`.
-+ Run :code:`activate-global-python-argcomplete`
-
-That is it. You're good to go. Autocomplete will be enabled working for vineyard-ctl.
-
-.. note::
-
-   In the bash shell, type :code:`vineyard-ctl sta` and press :code:`tab`, it will autocomplete
-   to :code:`vineyard-ctl start`
diff --git a/docs/notes/references/python-api.rst b/docs/notes/references/python-api.rst
index ed9193bf..75636234 100644
--- a/docs/notes/references/python-api.rst
+++ b/docs/notes/references/python-api.rst
@@ -126,3 +126,21 @@ Streams
 
 .. autoclass:: vineyard.io.recordbatch.RecordBatchStream
     :members:
+
+Interacting with the CSI Driver
+-------------------------------
+
+.. autofunction:: vineyard.csi.read
+.. autofunction:: vineyard.csi.write
+
+Vineyard Cli Tool
+-----------------
+
+You can also use the Python API to interact with internal
+`Vineyard Cli Tool`_.
+
+.. code-block:: bash
+
+    $ python -m vineyard.cli <command> [options]
+
+.. _Vineyard Cli Tool: https://v6d.io/notes/cloud-native/vineyardctl.html