diff --git a/ci/aws/aws-cluster.lokocfg.envsubst b/ci/aws/aws-cluster.lokocfg.envsubst index 1b7ad4663..a3cfdac89 100644 --- a/ci/aws/aws-cluster.lokocfg.envsubst +++ b/ci/aws/aws-cluster.lokocfg.envsubst @@ -275,7 +275,10 @@ component "httpbin" { certmanager_cluster_issuer = "letsencrypt-staging" } -component "aws-ebs-csi-driver" {} +component "aws-ebs-csi-driver" { + // Avoid proliferation of unused PVs. + reclaim_policy = "Delete" +} component "experimental-istio-operator" { enable_monitoring = true diff --git a/docs/configuration-reference/components/aws-ebs-csi-driver.md b/docs/configuration-reference/components/aws-ebs-csi-driver.md index 5d19b2559..24bfa5489 100644 --- a/docs/configuration-reference/components/aws-ebs-csi-driver.md +++ b/docs/configuration-reference/components/aws-ebs-csi-driver.md @@ -29,6 +29,7 @@ component "aws-ebs-csi-driver" { enable_volume_scheduling = true enable_volume_resizing = true enable_volume_snapshot = true + reclaim_policy = "Delete" node_affinity { key = "kubernetes.io/hostname" @@ -53,14 +54,15 @@ component "aws-ebs-csi-driver" { Table of all the arguments accepted by the component. -| Argument | Description | Default | Type | Required | -|--------------------------------|-------------------------------------------------------------------------------------------------------|---------------------------------------|------------------------------------------------------------------------------------------------------------------|----------| -| `enable_default_storage_class` | Use the storage class provided by the component as the default storage class. | `true` | bool | false | -| `enable_volume_scheduling` | Provision EBS volumes using PersistentVolumeClaim(PVC) dynamically. | `true` | bool | false | -| `enable_volume_resizing` | Expand the volume size after the initial provisioning. | `true` | bool | false | -| `enable_volume_snapshot` | Create Volume snapshots from an existing EBS volume for backup and restore. | `true` | bool | false | -| `node_affinity` | Node affinity for deploying the EBS CSI controller and EBS Snapshot controller pods. | - | `list(object({key = string, operator = string, values = list(string)}))` | false | -| `tolerations` | Tolerations that the EBS CSI Node, EBS CSI controller and EBS Snapshot controller pods will tolerate. | `tolerations { operator = "Exists" }` | `list(object({key = string, effect = string, operator = string, value = string, toleration_seconds = string }))` | false | +| Argument | Description | Default | Type | Required | +|--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------|------------------------------------------------------------------------------------------------------------------|----------| +| `enable_default_storage_class` | Use the storage class provided by the component as the default storage class. | `true` | bool | false | +| `enable_volume_scheduling` | Provision EBS volumes using PersistentVolumeClaim(PVC) dynamically. | `true` | bool | false | +| `enable_volume_resizing` | Expand the volume size after the initial provisioning. | `true` | bool | false | +| `enable_volume_snapshot` | Create Volume snapshots from an existing EBS volume for backup and restore. | `true` | bool | false | +| `node_affinity` | Node affinity for deploying the EBS CSI controller and EBS Snapshot controller pods. | - | `list(object({key = string, operator = string, values = list(string)}))` | false | +| `tolerations` | Tolerations that the EBS CSI Node, EBS CSI controller and EBS Snapshot controller pods will tolerate. | `tolerations { operator = "Exists" }` | `list(object({key = string, effect = string, operator = string, value = string, toleration_seconds = string }))` | false | +| `reclaim_policy` | Persistent volumes created with the storage class created by this component will have this reclaim policy. This field decides what happens to the volume after a user deletes a PVC. Valid values: `Retain`, `Recycle` and `Delete`. Read more in the [Kubernetes docs](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming). | `Retain` | string | false | ## Applying diff --git a/docs/configuration-reference/components/openebs-storage-class.md b/docs/configuration-reference/components/openebs-storage-class.md index 6eb630313..0f735a7f8 100644 --- a/docs/configuration-reference/components/openebs-storage-class.md +++ b/docs/configuration-reference/components/openebs-storage-class.md @@ -36,12 +36,15 @@ OpenEBS storage class component configuration example: component "openebs-storage-class" { # Optional arguments storage-class "openebs-replica1" { - replica_count = 1 + replica_count = 1 + reclaim_policy = "Delete" } + storage-class "openebs-replica3" { - replica_count = 3 - default = true - disks = [ + replica_count = 3 + default = true + reclaim_policy = "Retain" + disks = [ "blockdevice-0565dd2d566cab012b7bc35e54874d9f", "blockdevice-17901367ccd9e1ead797a7e233de8cc8", "blockdevice-1f4315cb4acbb4b0dbf5202adcdb70d8" @@ -54,11 +57,12 @@ component "openebs-storage-class" { Table of all the arguments accepted by the component. -| Argument | Description | Default | Type | Required | -|-----------------|----------------------------------------------------------------------------------------------------------------------------|:-------:|:------------:|:--------:| -| `replica_count` | Defines the number of cStor volume replicas. | 3 | number | false | -| `default` | Indicates whether the storage class is default or not. | false | bool | false | -| `disks` | List of selected unclaimed BlockDevice CRs which are unmounted and do not contain a filesystem in each participating node. | - | list(string) | false | +| Argument | Description | Default | Type | Required | +|------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------:|:------------:|:--------:| +| `replica_count` | Defines the number of cStor volume replicas. | 3 | number | false | +| `default` | Indicates whether the storage class is default or not. | false | bool | false | +| `disks` | List of selected unclaimed BlockDevice CRs which are unmounted and do not contain a filesystem in each participating node. | - | list(string) | false | +| `reclaim_policy` | Persistent volumes created with this storage class will have this reclaim policy. This field decides what happens to the volume after a user deletes a PVC. Valid values: `Retain`, `Recycle` and `Delete`. Read more in the [Kubernetes docs](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming). | `Retain` | string | false | ## Applying diff --git a/docs/configuration-reference/components/rook-ceph.md b/docs/configuration-reference/components/rook-ceph.md index 21c37f45f..53f501483 100644 --- a/docs/configuration-reference/components/rook-ceph.md +++ b/docs/configuration-reference/components/rook-ceph.md @@ -46,8 +46,9 @@ component "rook-ceph" { } storage_class { - enable = true - default = true + enable = true + default = true + reclaim_policy = "Delete" } } @@ -61,16 +62,17 @@ desired. Table of all the arguments accepted by the component. -| Argument | Description | Default | Type | Required | -|-------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------|:-------:|:---------------------------------------------------------------------------------------------------------------|:--------:| -| `namespace` | Namespace to deploy the Ceph cluster into. Must be the same as the rook operator. | "rook" | string | false | -| `monitor_count` | Number of Ceph monitors to deploy. An odd number like 3 or 5 is recommended which should also be sufficient for most cases. | 1 | number | false | -| `enable_toolbox` | Deploy the [toolbox pod](https://rook.io/docs/rook/master/ceph-toolbox.html) to debug and manage the Ceph cluster. | false | bool | false | -| `node_affinity` | Node affinity for deploying the Ceph cluster pods. | - | list(object({key = string, operator = string, values = list(string)})) | false | -| `toleration` | Tolerations that the Ceph cluster pods will tolerate. | - | list(object({key = string, effect = string, operator = string, value = string, toleration_seconds = string })) | false | -| `metadata_device` | Name of the device to store the metadata on each storage machine. **Note**: Provide just the name of the device and skip prefixing with `/dev/`. | - | string | false | -| `storage_class.enable` | Install Storage Class config. | false | bool | false | -| `storage_class.default` | Make this Storage Class as a default one. | false | bool | false | +| Argument | Description | Default | Type | Required | +|--------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------:|:---------------------------------------------------------------------------------------------------------------|:--------:| +| `namespace` | Namespace to deploy the Ceph cluster into. Must be the same as the rook operator. | "rook" | string | false | +| `monitor_count` | Number of Ceph monitors to deploy. An odd number like 3 or 5 is recommended which should also be sufficient for most cases. | 1 | number | false | +| `enable_toolbox` | Deploy the [toolbox pod](https://rook.io/docs/rook/master/ceph-toolbox.html) to debug and manage the Ceph cluster. | false | bool | false | +| `node_affinity` | Node affinity for deploying the Ceph cluster pods. | - | list(object({key = string, operator = string, values = list(string)})) | false | +| `toleration` | Tolerations that the Ceph cluster pods will tolerate. | - | list(object({key = string, effect = string, operator = string, value = string, toleration_seconds = string })) | false | +| `metadata_device` | Name of the device to store the metadata on each storage machine. **Note**: Provide just the name of the device and skip prefixing with `/dev/`. | - | string | false | +| `storage_class.enable` | Install Storage Class config. | false | bool | false | +| `storage_class.default` | Make this Storage Class as a default one. | false | bool | false | +| `storage_class.reclaim_policy` | Persistent volumes created with this storage class will have this reclaim policy. This field decides what happens to the volume after a user deletes a PVC. Valid values: `Retain`, `Recycle` and `Delete`. Read more in the [Kubernetes docs](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming). | `Retain` | string | false | ## Applying diff --git a/pkg/components/aws-ebs-csi-driver/component.go b/pkg/components/aws-ebs-csi-driver/component.go index 5d9d06749..217b772ac 100644 --- a/pkg/components/aws-ebs-csi-driver/component.go +++ b/pkg/components/aws-ebs-csi-driver/component.go @@ -47,7 +47,7 @@ storageClasses: storageclass.kubernetes.io/is-default-class: "true" {{ end }} volumeBindingMode: WaitForFirstConsumer - reclaimPolicy: Retain + reclaimPolicy: {{ .ReclaimPolicy }} {{- if .Tolerations }} tolerateAllTaints: false @@ -74,6 +74,7 @@ type component struct { EnableVolumeSnapshot bool `hcl:"enable_volume_snapshot,optional"` Tolerations []util.Toleration `hcl:"tolerations,block"` NodeAffinity []util.NodeAffinity `hcl:"node_affinity,block"` + ReclaimPolicy string `hcl:"reclaim_policy,optional"` TolerationsRaw string NodeAffinityRaw string @@ -88,6 +89,7 @@ func NewConfig() *component { EnableVolumeScheduling: true, EnableVolumeResizing: true, EnableVolumeSnapshot: true, + ReclaimPolicy: "Retain", } } diff --git a/pkg/components/aws-ebs-csi-driver/component_test.go b/pkg/components/aws-ebs-csi-driver/component_test.go index 019b71480..dda96c7f3 100644 --- a/pkg/components/aws-ebs-csi-driver/component_test.go +++ b/pkg/components/aws-ebs-csi-driver/component_test.go @@ -276,6 +276,26 @@ func TestConversion(t *testing.T) { //nolint:funlen jsonPath: `{.metadata.annotations.storageclass\.kubernetes\.io\/is\-default\-class}`, expected: "true", }, + { + name: "default_reclaim_policy", + inputConfig: `component "aws-ebs-csi-driver" {}`, + expectedManifestName: k8sutil.ObjectMetadata{ + Version: "storage.k8s.io/v1", Kind: "StorageClass", Name: "ebs-sc", + }, + jsonPath: "{.reclaimPolicy}", + expected: "Retain", + }, + { + name: "overridden_reclaim_policy", + inputConfig: `component "aws-ebs-csi-driver" { + reclaim_policy = "Delete" + }`, + expectedManifestName: k8sutil.ObjectMetadata{ + Version: "storage.k8s.io/v1", Kind: "StorageClass", Name: "ebs-sc", + }, + jsonPath: "{.reclaimPolicy}", + expected: "Delete", + }, } for _, tc := range testCases { diff --git a/pkg/components/openebs-storage-class/component.go b/pkg/components/openebs-storage-class/component.go index 6916672e5..99b432926 100644 --- a/pkg/components/openebs-storage-class/component.go +++ b/pkg/components/openebs-storage-class/component.go @@ -36,10 +36,11 @@ const ( // StorageClass represents single OpenEBS storage class with properties. type StorageClass struct { - Name string `hcl:"name,label"` - ReplicaCount int `hcl:"replica_count,optional"` - Default bool `hcl:"default,optional"` - Disks []string `hcl:"disks,optional"` + Name string `hcl:"name,label"` + ReplicaCount int `hcl:"replica_count,optional"` + Default bool `hcl:"default,optional"` + Disks []string `hcl:"disks,optional"` + ReclaimPolicy string `hcl:"reclaim_policy,optional"` } type component struct { @@ -55,7 +56,8 @@ func defaultStorageClass() StorageClass { // Make the storage class as default Default: true, // Default disks selection is empty - Disks: make([]string, 0), + Disks: make([]string, 0), + ReclaimPolicy: "Retain", } } @@ -97,13 +99,17 @@ func (c *component) LoadConfig(configBody *hcl.Body, evalContext *hcl.EvalContex func (c *component) validateConfig() error { maxDefaultStorageClass := 0 - for _, sc := range c.StorageClasses { + for i, sc := range c.StorageClasses { if sc.Default == true { maxDefaultStorageClass++ } if maxDefaultStorageClass > 1 { return fmt.Errorf("cannot have more than one default storage class") } + + if sc.ReclaimPolicy == "" { + c.StorageClasses[i].ReclaimPolicy = "Retain" + } } return nil diff --git a/pkg/components/openebs-storage-class/component_test.go b/pkg/components/openebs-storage-class/component_test.go index cb65a2041..58bdda6da 100644 --- a/pkg/components/openebs-storage-class/component_test.go +++ b/pkg/components/openebs-storage-class/component_test.go @@ -19,7 +19,9 @@ import ( "github.com/hashicorp/hcl/v2" + "github.com/kinvolk/lokomotive/pkg/components/internal/testutil" "github.com/kinvolk/lokomotive/pkg/components/util" + "github.com/kinvolk/lokomotive/pkg/k8sutil" ) func TestEmptyConfig(t *testing.T) { @@ -88,3 +90,61 @@ func testRenderManifest(t *testing.T, configHCL string) { t.Fatalf("Rendered manifests shouldn't be empty") } } + +func TestConversion(t *testing.T) { + testCases := []struct { + name string + inputConfig string + expectedManifestName k8sutil.ObjectMetadata + expected string + jsonPath string + }{ + { + name: "default_storage_class", + inputConfig: `component "openebs-storage-class" {}`, + expectedManifestName: k8sutil.ObjectMetadata{ + // openebs-cstor-disk-replica-3 is the name of default SC created by this component. + Version: "storage.k8s.io/v1", Kind: "StorageClass", Name: "openebs-cstor-disk-replica-3", + }, + jsonPath: "{.reclaimPolicy}", + expected: "Retain", + }, + { + name: "default_reclaim_policy", + inputConfig: `component "openebs-storage-class" { + storage-class "test" {} + }`, + expectedManifestName: k8sutil.ObjectMetadata{ + Version: "storage.k8s.io/v1", Kind: "StorageClass", Name: "test", + }, + jsonPath: "{.reclaimPolicy}", + expected: "Retain", + }, + { + name: "overridden_reclaim_policy", + inputConfig: `component "openebs-storage-class" { + storage-class "test" { + reclaim_policy = "Delete" + } + }`, + expectedManifestName: k8sutil.ObjectMetadata{ + Version: "storage.k8s.io/v1", Kind: "StorageClass", Name: "test", + }, + jsonPath: "{.reclaimPolicy}", + expected: "Delete", + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + component := NewConfig() + m := testutil.RenderManifests(t, component, Name, tc.inputConfig) + gotConfig := testutil.ConfigFromMap(t, m, tc.expectedManifestName) + + testutil.MatchJSONPathStringValue(t, gotConfig, tc.jsonPath, tc.expected) + }) + } +} diff --git a/pkg/components/openebs-storage-class/template.go b/pkg/components/openebs-storage-class/template.go index c612b3b24..31db31c1b 100644 --- a/pkg/components/openebs-storage-class/template.go +++ b/pkg/components/openebs-storage-class/template.go @@ -28,7 +28,7 @@ metadata: - name: ReplicaCount value: "{{ .ReplicaCount }}" provisioner: openebs.io/provisioner-iscsi -reclaimPolicy: Delete +reclaimPolicy: "{{ .ReclaimPolicy }}" ` const storagePoolTmpl = ` diff --git a/pkg/components/rook-ceph/component.go b/pkg/components/rook-ceph/component.go index 24f9fb3aa..e345e5a42 100644 --- a/pkg/components/rook-ceph/component.go +++ b/pkg/components/rook-ceph/component.go @@ -45,8 +45,9 @@ type component struct { // StorageClass provides struct to enable it or make it default. type StorageClass struct { - Enable bool `hcl:"enable,optional"` - Default bool `hcl:"default,optional"` + Enable bool `hcl:"enable,optional"` + Default bool `hcl:"default,optional"` + ReclaimPolicy string `hcl:"reclaim_policy,optional"` } // NewConfig returns new Rook Ceph component configuration with default values set. @@ -56,7 +57,7 @@ func NewConfig() *component { return &component{ Namespace: "rook", MonitorCount: 1, - StorageClass: &StorageClass{}, + StorageClass: &StorageClass{ReclaimPolicy: "Retain"}, } } diff --git a/pkg/components/rook-ceph/component_test.go b/pkg/components/rook-ceph/component_test.go index 4d7104cc7..929aac2e6 100644 --- a/pkg/components/rook-ceph/component_test.go +++ b/pkg/components/rook-ceph/component_test.go @@ -19,7 +19,9 @@ import ( "github.com/hashicorp/hcl/v2" + "github.com/kinvolk/lokomotive/pkg/components/internal/testutil" "github.com/kinvolk/lokomotive/pkg/components/util" + "github.com/kinvolk/lokomotive/pkg/k8sutil" ) func TestEmptyConfig(t *testing.T) { @@ -82,3 +84,54 @@ component "rook-ceph" { t.Fatalf("Rendered manifests shouldn't be empty") } } + +func TestConversion(t *testing.T) { + testCases := []struct { + name string + inputConfig string + expectedManifestName k8sutil.ObjectMetadata + expected string + jsonPath string + }{ + { + name: "default_reclaim_policy", + inputConfig: `component "rook-ceph" { + storage_class { + enable = true + } + }`, + expectedManifestName: k8sutil.ObjectMetadata{ + Version: "storage.k8s.io/v1", Kind: "StorageClass", Name: "rook-ceph-block", + }, + jsonPath: "{.reclaimPolicy}", + expected: "Retain", + }, + { + name: "overridden_reclaim_policy", + inputConfig: `component "rook-ceph" { + storage_class { + enable = true + reclaim_policy = "Delete" + } + }`, + expectedManifestName: k8sutil.ObjectMetadata{ + Version: "storage.k8s.io/v1", Kind: "StorageClass", Name: "rook-ceph-block", + }, + jsonPath: "{.reclaimPolicy}", + expected: "Delete", + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + component := NewConfig() + m := testutil.RenderManifests(t, component, Name, tc.inputConfig) + gotConfig := testutil.ConfigFromMap(t, m, tc.expectedManifestName) + + testutil.MatchJSONPathStringValue(t, gotConfig, tc.jsonPath, tc.expected) + }) + } +} diff --git a/pkg/components/rook-ceph/manifests.go b/pkg/components/rook-ceph/manifests.go index fa346010b..58170abf3 100644 --- a/pkg/components/rook-ceph/manifests.go +++ b/pkg/components/rook-ceph/manifests.go @@ -161,7 +161,7 @@ parameters: csi.storage.k8s.io/fstype: xfs # Delete the rbd volume when a PVC is deleted -reclaimPolicy: Delete +reclaimPolicy: {{ .StorageClass.ReclaimPolicy }} {{- end }} `,