From 93d15491499ed85cedf2633d45ece11120db9a6b Mon Sep 17 00:00:00 2001 From: Simon Hui Date: Fri, 2 Nov 2018 18:42:35 +0800 Subject: [PATCH 1/5] Support read specific label (kompose.volume.size) from named volume and apply to kubernetes supported volume size --- pkg/loader/compose/v3.go | 36 +++++++++++++++++++++++- pkg/transformer/kubernetes/kubernetes.go | 10 +++++-- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/pkg/loader/compose/v3.go b/pkg/loader/compose/v3.go index c479b2577..87057003c 100644 --- a/pkg/loader/compose/v3.go +++ b/pkg/loader/compose/v3.go @@ -422,11 +422,45 @@ func dockerComposeToKomposeMapping(composeObject *types.Config) (kobject.Kompose komposeObject.ServiceConfigs[normalizeServiceNames(name)] = serviceConfig } - handleVolume(&komposeObject) + handleV3Volume(&komposeObject, &composeObject.Volumes) return komposeObject, nil } +func handleV3Volume(komposeObject *kobject.KomposeObject, volumes *map[string]types.VolumeConfig) { + for name := range komposeObject.ServiceConfigs { + // retrieve volumes of service + vols, err := retrieveVolume(name, *komposeObject) + if err != nil { + errors.Wrap(err, "could not retrieve vvolume") + } + for volName, vol := range vols { + size := getV3VolumeSize(vol.VolumeName, volumes) + if len(size) > 0 { + // We can't assign value to struct field in map while iterating over it, so temporary variable `temp` is used here + var temp = vols[volName] + temp.PVCSize = size + vols[volName] = temp + } + } + // We can't assign value to struct field in map while iterating over it, so temporary variable `temp` is used here + var temp = komposeObject.ServiceConfigs[name] + temp.Volumes = vols + komposeObject.ServiceConfigs[name] = temp + } +} + +func getV3VolumeSize(name string, volumes *map[string]types.VolumeConfig) string { + if volume, ok := (*volumes)[name]; ok { + for key, value := range volume.Labels { + if key == "kompose.volume.size" { + return value + } + } + } + return "" +} + func mergeComposeObject(oldCompose *types.Config, newCompose *types.Config) (*types.Config, error) { if oldCompose == nil || newCompose == nil { return nil, fmt.Errorf("Merge multiple compose error, compose config is nil") diff --git a/pkg/transformer/kubernetes/kubernetes.go b/pkg/transformer/kubernetes/kubernetes.go index c8618f78a..2a213df30 100644 --- a/pkg/transformer/kubernetes/kubernetes.go +++ b/pkg/transformer/kubernetes/kubernetes.go @@ -578,9 +578,13 @@ func (k *Kubernetes) ConfigVolumes(name string, service kobject.ServiceConfig) ( if volume.VFrom == "" { defaultSize := PVCRequestSize - for key, value := range service.Labels { - if key == "kompose.volume.size" { - defaultSize = value + if len(volume.PVCSize) > 0 { + defaultSize = volume.PVCSize + } else { + for key, value := range service.Labels { + if key == "kompose.volume.size" { + defaultSize = value + } } } From c83914e6c787df1d65904cc59afb7d06c9df434e Mon Sep 17 00:00:00 2001 From: Simon Hui Date: Tue, 6 Nov 2018 12:30:38 +0800 Subject: [PATCH 2/5] Fix the PVC size in log message when deploy Kubernetes --- pkg/transformer/kubernetes/kubernetes.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/transformer/kubernetes/kubernetes.go b/pkg/transformer/kubernetes/kubernetes.go index 2a213df30..166682131 100644 --- a/pkg/transformer/kubernetes/kubernetes.go +++ b/pkg/transformer/kubernetes/kubernetes.go @@ -1018,7 +1018,9 @@ func (k *Kubernetes) Deploy(komposeObject kobject.KomposeObject, opt kobject.Con if err != nil { return err } - log.Infof("Successfully created PersistentVolumeClaim: %s of size %s. If your cluster has dynamic storage provisioning, you don't have to do anything. Otherwise you have to create PersistentVolume to make PVC work", t.Name, PVCRequestSize) + storage := t.Spec.Resources.Requests[api.ResourceStorage] + capacity := storage.String() + log.Infof("Successfully created PersistentVolumeClaim: %s of size %s. If your cluster has dynamic storage provisioning, you don't have to do anything. Otherwise you have to create PersistentVolume to make PVC work", t.Name, capacity) case *extensions.Ingress: _, err := client.Ingress(namespace).Create(t) if err != nil { From 22669f50369ae1b55b523f6e1f25cd61a012f4f4 Mon Sep 17 00:00:00 2001 From: Simon Hui Date: Tue, 6 Nov 2018 19:46:19 +0800 Subject: [PATCH 3/5] Skip creation of PersistentVolumeClaim if it is already created in the same kubernetes deploy --- pkg/transformer/kubernetes/kubernetes.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pkg/transformer/kubernetes/kubernetes.go b/pkg/transformer/kubernetes/kubernetes.go index 166682131..6c5716efd 100644 --- a/pkg/transformer/kubernetes/kubernetes.go +++ b/pkg/transformer/kubernetes/kubernetes.go @@ -982,6 +982,8 @@ func (k *Kubernetes) Deploy(komposeObject kobject.KomposeObject, opt kobject.Con return err } + pvcCreatedSet := make(map[string]bool) + log.Infof("Deploying application in %q namespace", namespace) for _, v := range objects { @@ -1014,13 +1016,18 @@ func (k *Kubernetes) Deploy(komposeObject kobject.KomposeObject, opt kobject.Con } log.Infof("Successfully created Service: %s", t.Name) case *api.PersistentVolumeClaim: - _, err := client.PersistentVolumeClaims(namespace).Create(t) - if err != nil { - return err + if pvcCreatedSet[t.Name] { + log.Infof("Skip creation of PersistentVolumeClaim as it is already created: %s", t.Name) + } else { + _, err := client.PersistentVolumeClaims(namespace).Create(t) + if err != nil { + return err + } + pvcCreatedSet[t.Name] = true + storage := t.Spec.Resources.Requests[api.ResourceStorage] + capacity := storage.String() + log.Infof("Successfully created PersistentVolumeClaim: %s of size %s. If your cluster has dynamic storage provisioning, you don't have to do anything. Otherwise you have to create PersistentVolume to make PVC work", t.Name, capacity) } - storage := t.Spec.Resources.Requests[api.ResourceStorage] - capacity := storage.String() - log.Infof("Successfully created PersistentVolumeClaim: %s of size %s. If your cluster has dynamic storage provisioning, you don't have to do anything. Otherwise you have to create PersistentVolume to make PVC work", t.Name, capacity) case *extensions.Ingress: _, err := client.Ingress(namespace).Create(t) if err != nil { From a475e07be63f9b52ba6902329a52706066c0c2e9 Mon Sep 17 00:00:00 2001 From: Simon Hui Date: Wed, 14 Nov 2018 15:58:17 +0800 Subject: [PATCH 4/5] Add selector to PersistentVolumeClaim only when specific label (kompose.volume.selector) is used in named volume --- pkg/kobject/kobject.go | 19 ++++++++++--------- pkg/loader/compose/v3.go | 16 +++++++++++----- pkg/transformer/kubernetes/kubernetes.go | 10 ++++++++-- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/pkg/kobject/kobject.go b/pkg/kobject/kobject.go index a6166e997..540c8b2be 100644 --- a/pkg/kobject/kobject.go +++ b/pkg/kobject/kobject.go @@ -143,13 +143,14 @@ type Ports struct { // Volumes holds the volume struct of container type Volumes struct { - SvcName string // Service name to which volume is linked - MountPath string // Mountpath extracted from docker-compose file - VFrom string // denotes service name from which volume is coming - VolumeName string // name of volume if provided explicitly - Host string // host machine address - Container string // Mountpath - Mode string // access mode for volume - PVCName string // name of PVC - PVCSize string // PVC size + SvcName string // Service name to which volume is linked + MountPath string // Mountpath extracted from docker-compose file + VFrom string // denotes service name from which volume is coming + VolumeName string // name of volume if provided explicitly + Host string // host machine address + Container string // Mountpath + Mode string // access mode for volume + PVCName string // name of PVC + PVCSize string // PVC size + SelectorValue string // Value of the label selector } diff --git a/pkg/loader/compose/v3.go b/pkg/loader/compose/v3.go index 87057003c..e0b3d45e0 100644 --- a/pkg/loader/compose/v3.go +++ b/pkg/loader/compose/v3.go @@ -435,11 +435,12 @@ func handleV3Volume(komposeObject *kobject.KomposeObject, volumes *map[string]ty errors.Wrap(err, "could not retrieve vvolume") } for volName, vol := range vols { - size := getV3VolumeSize(vol.VolumeName, volumes) - if len(size) > 0 { + size, selector := getV3VolumeLabels(vol.VolumeName, volumes) + if len(size) > 0 || len(selector) > 0 { // We can't assign value to struct field in map while iterating over it, so temporary variable `temp` is used here var temp = vols[volName] temp.PVCSize = size + temp.SelectorValue = selector vols[volName] = temp } } @@ -450,15 +451,20 @@ func handleV3Volume(komposeObject *kobject.KomposeObject, volumes *map[string]ty } } -func getV3VolumeSize(name string, volumes *map[string]types.VolumeConfig) string { +func getV3VolumeLabels(name string, volumes *map[string]types.VolumeConfig) (string, string) { + size, selector := "", "" + if volume, ok := (*volumes)[name]; ok { for key, value := range volume.Labels { if key == "kompose.volume.size" { - return value + size = value + } else if key == "kompose.volume.selector" { + selector = value } } } - return "" + + return size, selector } func mergeComposeObject(oldCompose *types.Config, newCompose *types.Config) (*types.Config, error) { diff --git a/pkg/transformer/kubernetes/kubernetes.go b/pkg/transformer/kubernetes/kubernetes.go index 6c5716efd..ef0f29bb4 100644 --- a/pkg/transformer/kubernetes/kubernetes.go +++ b/pkg/transformer/kubernetes/kubernetes.go @@ -373,7 +373,7 @@ func (k *Kubernetes) initIngress(name string, service kobject.ServiceConfig, por } // CreatePVC initializes PersistentVolumeClaim -func (k *Kubernetes) CreatePVC(name string, mode string, size string) (*api.PersistentVolumeClaim, error) { +func (k *Kubernetes) CreatePVC(name string, mode string, size string, selectorValue string) (*api.PersistentVolumeClaim, error) { volSize, err := resource.ParseQuantity(size) if err != nil { return nil, errors.Wrap(err, "resource.ParseQuantity failed, Error parsing size") @@ -397,6 +397,12 @@ func (k *Kubernetes) CreatePVC(name string, mode string, size string) (*api.Pers }, } + if len(selectorValue) > 0 { + pvc.Spec.Selector = &unversioned.LabelSelector{ + MatchLabels: transformer.ConfigLabels(selectorValue), + } + } + if mode == "ro" { pvc.Spec.AccessModes = []api.PersistentVolumeAccessMode{api.ReadOnlyMany} } else { @@ -588,7 +594,7 @@ func (k *Kubernetes) ConfigVolumes(name string, service kobject.ServiceConfig) ( } } - createdPVC, err := k.CreatePVC(volumeName, volume.Mode, defaultSize) + createdPVC, err := k.CreatePVC(volumeName, volume.Mode, defaultSize, volume.SelectorValue) if err != nil { return nil, nil, nil, errors.Wrap(err, "k.CreatePVC failed") From 11d497214ab62c65397394f8667c62dfb1ae6bda Mon Sep 17 00:00:00 2001 From: Simon Hui Date: Wed, 28 Nov 2018 17:14:34 +0800 Subject: [PATCH 5/5] Add test case to named-volume for the new labels --- script/test/cmd/tests.sh | 4 + .../named-volume/docker-compose-v3.yml | 16 ++ .../named-volume/output-k8s-v3.json | 157 ++++++++++++++++++ 3 files changed, 177 insertions(+) create mode 100644 script/test/fixtures/volume-mounts/named-volume/docker-compose-v3.yml create mode 100644 script/test/fixtures/volume-mounts/named-volume/output-k8s-v3.json diff --git a/script/test/cmd/tests.sh b/script/test/cmd/tests.sh index 193cda377..65312fbfd 100755 --- a/script/test/cmd/tests.sh +++ b/script/test/cmd/tests.sh @@ -185,6 +185,10 @@ cmd="kompose -f $KOMPOSE_ROOT/script/test/fixtures/volume-mounts/named-volume/do sed -e "s;%VERSION%;$version;g" -e "s;%CMD%;$cmd;g" $KOMPOSE_ROOT/script/test/fixtures/volume-mounts/named-volume/output-k8s-template.json > /tmp/output-k8s.json convert::expect_success "$cmd" "/tmp/output-k8s.json" +cmd="kompose -f $KOMPOSE_ROOT/script/test/fixtures/volume-mounts/named-volume/docker-compose-v3.yml convert --stdout -j" +sed -e "s;%VERSION%;$version;g" -e "s;%CMD%;$cmd;g" $KOMPOSE_ROOT/script/test/fixtures/volume-mounts/named-volume/output-k8s-v3.json > /tmp/output-k8s.json +convert::expect_success "$cmd" "/tmp/output-k8s.json" + # openshift test cmd="kompose --provider=openshift -f $KOMPOSE_ROOT/script/test/fixtures/volume-mounts/named-volume/docker-compose.yml convert --stdout -j" sed -e "s;%VERSION%;$version;g" -e "s;%CMD%;$cmd;g" $KOMPOSE_ROOT/script/test/fixtures/volume-mounts/named-volume/output-os-template.json > /tmp/output-os.json diff --git a/script/test/fixtures/volume-mounts/named-volume/docker-compose-v3.yml b/script/test/fixtures/volume-mounts/named-volume/docker-compose-v3.yml new file mode 100644 index 000000000..7bce38959 --- /dev/null +++ b/script/test/fixtures/volume-mounts/named-volume/docker-compose-v3.yml @@ -0,0 +1,16 @@ +version: '3' +services: + db: + image: postgres:10.1 + ports: + - "5432" + labels: + kompose.volume.size: 200Mi + volumes: + - db-data:/var/lib/postgresql/data + - db-config:/var/lib/postgresql/config +volumes: + db-data: + labels: + kompose.volume.selector: db-data-dev + kompose.volume.size: 500Mi \ No newline at end of file diff --git a/script/test/fixtures/volume-mounts/named-volume/output-k8s-v3.json b/script/test/fixtures/volume-mounts/named-volume/output-k8s-v3.json new file mode 100644 index 000000000..2bf78a454 --- /dev/null +++ b/script/test/fixtures/volume-mounts/named-volume/output-k8s-v3.json @@ -0,0 +1,157 @@ +{ + "kind": "List", + "apiVersion": "v1", + "metadata": {}, + "items": [ + { + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "db", + "creationTimestamp": null, + "labels": { + "io.kompose.service": "db" + }, + "annotations": { + "kompose.cmd": "%CMD%", + "kompose.version": "%VERSION%", + "kompose.volume.size": "200Mi" + } + }, + "spec": { + "ports": [ + { + "name": "5432", + "port": 5432, + "targetPort": 5432 + } + ], + "selector": { + "io.kompose.service": "db" + } + }, + "status": { + "loadBalancer": {} + } + }, + { + "kind": "Deployment", + "apiVersion": "extensions/v1beta1", + "metadata": { + "name": "db", + "creationTimestamp": null, + "labels": { + "io.kompose.service": "db" + }, + "annotations": { + "kompose.cmd": "%CMD%", + "kompose.version": "%VERSION%", + "kompose.volume.size": "200Mi" + } + }, + "spec": { + "replicas": 1, + "template": { + "metadata": { + "creationTimestamp": null, + "labels": { + "io.kompose.service": "db" + } + }, + "spec": { + "volumes": [ + { + "name": "db-data", + "persistentVolumeClaim": { + "claimName": "db-data" + } + }, + { + "name": "db-config", + "persistentVolumeClaim": { + "claimName": "db-config" + } + } + ], + "containers": [ + { + "name": "db", + "image": "postgres:10.1", + "ports": [ + { + "containerPort": 5432 + } + ], + "resources": {}, + "volumeMounts": [ + { + "name": "db-data", + "mountPath": "/var/lib/postgresql/data" + }, + { + "name": "db-config", + "mountPath": "/var/lib/postgresql/config" + } + ] + } + ], + "restartPolicy": "Always" + } + }, + "strategy": { + "type": "Recreate" + } + }, + "status": {} + }, + { + "kind": "PersistentVolumeClaim", + "apiVersion": "v1", + "metadata": { + "name": "db-data", + "creationTimestamp": null, + "labels": { + "io.kompose.service": "db-data" + } + }, + "spec": { + "accessModes": [ + "ReadWriteOnce" + ], + "selector": { + "matchLabels": { + "io.kompose.service": "db-data-dev" + } + }, + "resources": { + "requests": { + "storage": "500Mi" + } + } + }, + "status": {} + }, + { + "kind": "PersistentVolumeClaim", + "apiVersion": "v1", + "metadata": { + "name": "db-config", + "creationTimestamp": null, + "labels": { + "io.kompose.service": "db-config" + } + }, + "spec": { + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "200Mi" + } + } + }, + "status": {} + } + ] +}