Skip to content

Commit

Permalink
provide option to set pluginpath for cephfs
Browse files Browse the repository at this point in the history 
Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
  • Loading branch information
@Madhu-1
Madhu-1 committed Jul 11, 2019
1 parent fa68c35 commit 8625047
Show file tree
Hide file tree
Showing 6 changed files with 39 additions and 49 deletions.
29 changes: 12 additions & 17 deletions cmd/cephcsi.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"flag"
"os"
"path"
"path/filepath"
"strings"

"github.com/ceph/ceph-csi/pkg/cephfs"
Expand All @@ -45,6 +46,7 @@ var (
instanceID = flag.String("instanceid", "", "Unique ID distinguishing this instance of Ceph CSI among other"+
" instances, when sharing Ceph clusters across CSI instances for provisioning")
metadataStorage = flag.String("metadatastorage", "", "metadata persistence method [node|k8s_configmap]")
pluginPath = flag.String("pluginpath", "/var/lib/kubelet/plugins/", "the location of cephcsi plugin")

// rbd related flags
containerized = flag.Bool("containerized", true, "whether run as containerized")
Expand Down Expand Up @@ -105,31 +107,24 @@ func main() {
if err != nil {
klog.Fatalln(err) // calls exit
}
csipluginPath := filepath.Join(*pluginPath, dname)
if *metadataStorage != "" {
cp, err = util.CreatePersistanceStorage(
csipluginPath, *metadataStorage, *pluginPath)
if err != nil {
os.Exit(1)
}
}

klog.Infof("Starting driver type: %v with name: %v", driverType, dname)
switch driverType {
case rbdType:
rbd.PluginFolder += dname
if *metadataStorage != "" {
cp, err = util.CreatePersistanceStorage(
rbd.PluginFolder, *metadataStorage, dname)
if err != nil {
os.Exit(1)
}
}
driver := rbd.NewDriver()
driver.Run(dname, *nodeID, *endpoint, *instanceID, *containerized, cp)

case cephfsType:
cephfs.PluginFolder += dname
if *metadataStorage != "" {
cp, err = util.CreatePersistanceStorage(
cephfs.PluginFolder, *metadataStorage, dname)
if err != nil {
os.Exit(1)
}
}
driver := cephfs.NewDriver()
driver.Run(dname, *nodeID, *endpoint, *volumeMounter, *mountCacheDir, *instanceID, cp)
driver.Run(dname, *nodeID, *endpoint, *volumeMounter, *mountCacheDir, *instanceID, csipluginPath, cp)

default:
klog.Fatalln("invalid volume type", vtype) // calls exit
Expand Down
38 changes: 20 additions & 18 deletions docs/deploy-cephfs.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,18 @@ make image-cephcsi

**Available command line arguments:**

Option | Default value | Description
--------------------|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
`--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket
`--drivername` | `cephfs.csi.ceph.com` | Name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value)
`--nodeid` | _empty_ | This node's ID
| `--type` | _empty_ | Driver type `[rbd | cephfs]` If the driver type is set to `rbd` it will act as a `rbd plugin` or if it's set to `cephfs` will act as a `cephfs plugin`
`--volumemounter` | _empty_ | Default volume mounter. Available options are `kernel` and `fuse`. This is the mount method used if volume parameters don't specify otherwise. If left unspecified, the driver will first probe for `ceph-fuse` in system's path and will choose Ceph kernel client if probing failed.
`--mountcachedir` | _empty_ | Volume mount cache info save dir. If left unspecified, the dirver will not record mount info, or it will save mount info and when driver restart it will remount volume it cached.
`--instanceid` | "default" | Unique ID distinguishing this instance of Ceph CSI among other instances, when sharing Ceph clusters across CSI instances for provisioning
`--metadatastorage` | _empty_ | Points to where older (1.0.0 or older plugin versions) metadata about provisioned volumes are kept, as file or in as k8s configmap (`node` or `k8s_configmap` respectively)
| Option | Default value | Description |
| ----------------- | --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket |
| `--drivername` | `cephfs.csi.ceph.com` | Name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) |
| `--nodeid` | _empty_ | This node's ID |
| `--type` | _empty_ | Driver type `[rbd | cephfs]` If the driver type is set to `rbd` it will act as a `rbd plugin` or if it's set to `cephfs` will act as a `cephfs plugin` |
| `--volumemounter` | _empty_ | Default volume mounter. Available options are `kernel` and `fuse`. This is the mount method used if volume parameters don't specify otherwise. If left unspecified, the driver will first probe for `ceph-fuse` in system's path and will choose Ceph kernel client if probing failed. |
| `--mountcachedir` | _empty_ | Volume mount cache info save dir. If left unspecified, the dirver will not record mount info, or it will save mount info and when driver restart it will remount volume it cached. |
| `--instanceid` | "default" | Unique ID distinguishing this instance of Ceph CSI among other instances, when sharing Ceph clusters across CSI instances for provisioning |
| `--pluginpath` | "/var/lib/kubelet/plugins/" | The location of cephcsi plugin on host |

| `--metadatastorage` | _empty_ | Points to where older (1.0.0 or older plugin versions) metadata about provisioned volumes are kept, as file or in as k8s configmap (`node` or `k8s_configmap` respectively) |

**Available environmental variables:**

Expand All @@ -50,14 +52,14 @@ is used to define in which namespace you want the configmaps to be stored

**Available volume parameters:**

Parameter | Required | Description
----------------------------------------------------------------------------------------------------|--------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
`clusterID` | yes | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use
`fsName` | yes | CephFS filesystem name into which the volume shall be created
`mounter` | no | Mount method to be used for this volume. Available options are `kernel` for Ceph kernel client and `fuse` for Ceph FUSE driver. Defaults to "default mounter", see command line arguments.
`pool` | yes | Ceph pool into which the volume shall be created
`csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-stage-secret-name` | for Kubernetes | Name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value
`csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-stage-secret-namespace` | for Kubernetes | Namespaces of the above Secret objects
| Parameter | Required | Description |
| --------------------------------------------------------------------------------------------------- | -------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `clusterID` | yes | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use |
| `fsName` | yes | CephFS filesystem name into which the volume shall be created |
| `mounter` | no | Mount method to be used for this volume. Available options are `kernel` for Ceph kernel client and `fuse` for Ceph FUSE driver. Defaults to "default mounter", see command line arguments. |
| `pool` | yes | Ceph pool into which the volume shall be created |
| `csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-stage-secret-name` | for Kubernetes | Name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value |
| `csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-stage-secret-namespace` | for Kubernetes | Namespaces of the above Secret objects |

**NOTE:** An accompanying CSI configuration file, needs to be provided to the
running pods. Refer to [Creating CSI configuration](../examples/README.md#creating-csi-configuration)
Expand Down
5 changes: 3 additions & 2 deletions pkg/cephfs/driver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ const (
)

// PluginFolder defines the location of ceph plugin
var PluginFolder = "/var/lib/kubelet/plugins/"
var PluginFolder = ""

// Driver contains the default identity,node and controller struct
type Driver struct {
Expand Down Expand Up @@ -93,10 +93,11 @@ func NewNodeServer(d *csicommon.CSIDriver) *NodeServer {

// Run start a non-blocking grpc controller,node and identityserver for
// ceph CSI driver which can serve multiple parallel requests
func (fs *Driver) Run(driverName, nodeID, endpoint, volumeMounter, mountCacheDir, instanceID string, cachePersister util.CachePersister) {
func (fs *Driver) Run(driverName, nodeID, endpoint, volumeMounter, mountCacheDir, instanceID, pluginPath string, cachePersister util.CachePersister) {
klog.Infof("Driver: %v version: %v", driverName, version)

// Configuration
PluginFolder = pluginPath

if err := loadAvailableMounters(); err != nil {
klog.Fatalf("cephfs: failed to load ceph mounters: %v", err)
Expand Down
3 changes: 0 additions & 3 deletions pkg/rbd/rbd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,6 @@ type Driver struct {
var (
version = "1.0.0"

// PluginFolder defines the location of ceph plugin
PluginFolder = "/var/lib/kubelet/plugins/"

// CSIInstanceID is the instance ID that is unique to an instance of CSI, used when sharing
// ceph clusters across CSI instances, to differentiate omap names per CSI instance
CSIInstanceID = "default"
Expand Down
9 changes: 2 additions & 7 deletions pkg/util/cachepersister.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,6 @@ import (
"k8s.io/klog"
)

const (
// PluginFolder defines location of plugins
PluginFolder = "/var/lib/kubelet/plugins"
)

// ForAllFunc is a unary predicate for visiting all cache entries
// matching the `pattern' in CachePersister's ForAll function.
type ForAllFunc func(identifier string) error
Expand All @@ -45,7 +40,7 @@ type CachePersister interface {
}

// NewCachePersister returns CachePersister based on store
func NewCachePersister(metadataStore, driverName string) (CachePersister, error) {
func NewCachePersister(metadataStore, pluginPath string) (CachePersister, error) {
if metadataStore == "k8s_configmap" {
klog.Infof("cache-perister: using kubernetes configmap as metadata cache persister")
k8scm := &K8sCMCache{}
Expand All @@ -55,7 +50,7 @@ func NewCachePersister(metadataStore, driverName string) (CachePersister, error)
} else if metadataStore == "node" {
klog.Infof("cache-persister: using node as metadata cache persister")
nc := &NodeCache{}
nc.BasePath = PluginFolder + "/" + driverName
nc.BasePath = pluginPath
nc.CacheDir = "controller"
return nc, nil
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/util/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ func roundUpSize(volumeSizeBytes, allocationUnitBytes int64) int64 {
}

// CreatePersistanceStorage creates storage path and initializes new cache
func CreatePersistanceStorage(sPath, metaDataStore, driverName string) (CachePersister, error) {
func CreatePersistanceStorage(sPath, metaDataStore, pluginPath string) (CachePersister, error) {
var err error
if err = createPersistentStorage(path.Join(sPath, "controller")); err != nil {
klog.Errorf("failed to create persistent storage for controller: %v", err)
Expand All @@ -60,7 +60,7 @@ func CreatePersistanceStorage(sPath, metaDataStore, driverName string) (CachePer
return nil, err
}

cp, err := NewCachePersister(metaDataStore, driverName)
cp, err := NewCachePersister(metaDataStore, pluginPath)
if err != nil {
klog.Errorf("failed to define cache persistence method: %v", err)
return nil, err
Expand Down

0 comments on commit 8625047

Please sign in to comment.