Google Cloud Storage Import Support

README.md

@@ -76,7 +76,7 @@ Create a DataVolume and populate it with data from an http source
 $ kubectl create -f https://raw.githubusercontent.com/kubevirt/containerized-data-importer/$VERSION/manifests/example/import-kubevirt-datavolume.yaml
 ```
 
-There are quite a few examples in the [example manifests](https://github.com/kubevirt/containerized-data-importer/tree/main/manifests/example), check them out as a reference to create DataVolumes from additional sources like registries, S3 and your local system.
+There are quite a few examples in the [example manifests](https://github.com/kubevirt/containerized-data-importer/tree/main/manifests/example), check them out as a reference to create DataVolumes from additional sources like registries, S3, GCS and your local system.
 
 ## Hack it
 

api/openapi-spec/swagger.json

@@ -5489,12 +5489,15 @@
     }
    },
    "v1beta1.DataVolumeSource": {
-    "description": "DataVolumeSource represents the source for our Data Volume, this can be HTTP, Imageio, S3, Registry or an existing PVC",
+    "description": "DataVolumeSource represents the source for our Data Volume, this can be HTTP, Imageio, S3, GCS, Registry or an existing PVC",
     "type": "object",
     "properties": {
      "blank": {
       "$ref": "#/definitions/v1beta1.DataVolumeBlankImage"
      },
+     "gcs": {
+      "$ref": "#/definitions/v1beta1.DataVolumeSourceGCS"
+     },
      "http": {
       "$ref": "#/definitions/v1beta1.DataVolumeSourceHTTP"
      },
@@ -5521,6 +5524,24 @@
      }
     }
    },
+   "v1beta1.DataVolumeSourceGCS": {
+    "description": "DataVolumeSourceGCS provides the parameters to create a Data Volume from an GCS source",
+    "type": "object",
+    "required": [
+     "url"
+    ],
+    "properties": {
+     "secretRef": {
+      "description": "SecretRef provides the secret reference needed to access the GCS source",
+      "type": "string"
+     },
+     "url": {
+      "description": "URL is the url of the GCS source",
+      "type": "string",
+      "default": ""
+     }
+    }
+   },
    "v1beta1.DataVolumeSourceHTTP": {
     "description": "DataVolumeSourceHTTP can be either an http or https endpoint, with an optional basic auth user name and password, and an optional configmap containing additional CAs",
     "type": "object",

cmd/cdi-importer/importer.go

@@ -236,6 +236,7 @@ func newDataSource(source string, contentType string, volumeMode v1.PersistentVo
 	ep, _ := util.ParseEnvVar(common.ImporterEndpoint, false)
 	acc, _ := util.ParseEnvVar(common.ImporterAccessKeyID, false)
 	sec, _ := util.ParseEnvVar(common.ImporterSecretKey, false)
+	keyf, _ := util.ParseEnvVar(common.ImporterGoogleCredentialFileVar, false)
 	diskID, _ := util.ParseEnvVar(common.ImporterDiskID, false)
 	uuid, _ := util.ParseEnvVar(common.ImporterUUID, false)
 	backingFile, _ := util.ParseEnvVar(common.ImporterBackingFile, false)
@@ -269,6 +270,12 @@ func newDataSource(source string, contentType string, volumeMode v1.PersistentVo
 			errorCannotConnectDataSource(err, "s3")
 		}
 		return ds
+	case cc.SourceGCS:
+		ds, err := importer.NewGCSDataSource(ep, keyf)
+		if err != nil {
+			errorCannotConnectDataSource(err, "gcs")
+		}
+		return ds
 	case cc.SourceVDDK:
 		ds, err := importer.NewVDDKDataSource(ep, acc, sec, thumbprint, uuid, backingFile, currentCheckpoint, previousCheckpoint, finalCheckpoint, volumeMode)
 		if err != nil {

doc/annotations.md

@@ -5,6 +5,7 @@ All annotations associated with Containerized Data Importer (CDI) have a prefix
 Source describes the type of data source CDI will be collecting the data from. Based on the value of source, additional annotations may be required to successfully import the data. The full annotation for source is: cdi.kubevirt.io/storage.import.source. The following values are currently available:
 * http
 * S3
+* GCS
 * registry
 * none (don't import, but create data based on the contentType annotation)
 

doc/datavolumes.md

@@ -38,8 +38,8 @@ The following statuses are possible.
 
 ## Source 
 
-### HTTP/S3/Registry source
-DataVolumes are an abstraction on top of the annotations one can put on PVCs to trigger CDI. As such DVs have the notion of a 'source' that allows one to specify the source of the data. To import data from an external source, the source has to be either 'http' ,'S3' or 'registry'. If your source requires authentication, you can also pass in a `secretRef` to a Kubernetes [Secret](../manifests/example/endpoint-secret.yaml) containing the authentication information.  TLS certificates for https/registry sources may be specified in a [ConfigMap](../manifests/example/cert-configmap.yaml) and referenced by `certConfigMap`.  `secretRef` and `certConfigMap` must be in the same namespace as the DataVolume.
+### HTTP/S3/GCS/Registry source
+DataVolumes are an abstraction on top of the annotations one can put on PVCs to trigger CDI. As such DVs have the notion of a 'source' that allows one to specify the source of the data. To import data from an external source, the source has to be either 'http' ,'S3', 'GCS' or 'registry'. If your source requires authentication, you can also pass in a `secretRef` to a Kubernetes [Secret](../manifests/example/endpoint-secret.yaml) containing the authentication information.  TLS certificates for https/registry sources may be specified in a [ConfigMap](../manifests/example/cert-configmap.yaml) and referenced by `certConfigMap`.  `secretRef` and `certConfigMap` must be in the same namespace as the DataVolume.
 
 ```yaml
 apiVersion: cdi.kubevirt.io/v1beta1
@@ -49,7 +49,7 @@ metadata:
 spec:
   source:
       http:
-         url: "https://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img" # Or S3
+         url: "https://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img" # S3 or GCS
          secretRef: "" # Optional
          certConfigMap: "" # Optional
   pvc:
@@ -60,6 +60,7 @@ spec:
         storage: "64Mi"
 ```
 [Get example](../manifests/example/import-kubevirt-datavolume.yaml)
+[Get GCS example](../manifests/example/import-kubevirt-datavolume-gcs.yaml)
 [Get secret example](../manifests/example/endpoint-secret.yaml)
 [Get certificate example](../manifests/example/cert-configmap.yaml)
 
@@ -449,7 +450,7 @@ metadata:
 spec:
   source:
       http:
-         url: "https://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img" # Or S3
+         url: "https://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img" # S3 or GCS
          secretRef: "" # Optional
          certConfigMap: "" # Optional
   pvc:

doc/design.md

@@ -73,6 +73,8 @@ The controller scans PVCs within its namespace by looking for specific annotatio
 - [import-kubevirt-block-datavolume.yaml](../manifests/example/import-kubevirt-block-datavolume.yaml)
 - [import-kubevirt-datavolume-secret.yaml](../manifests/example/import-kubevirt-datavolume-secret.yaml)
 - [import-kubevirt-datavolume.yaml](../manifests/example/import-kubevirt-datavolume.yaml)
+- [import-kubevirt-datavolume-gcs-secret.yaml](../manifests/example/import-kubevirt-datavolume-gcs-secret.yaml)
+- [import-kubevirt-datavolume-gcs.yaml](../manifests/example/import-kubevirt-datavolume-gcs.yaml)
 - [registry-image-block-datavolume.yaml](../manifests/example/registry-image-block-datavolume.yaml)
 - [registry-image-datavolume.yaml](../manifests/example/registry-image-datavolume.yaml)
 - [upload-block-datavolume-token.yaml](../manifests/example/upload-block-datavolume-token.yaml)

doc/supported_operations.md

@@ -4,7 +4,7 @@ The Containerized Data Importer (CDI) supports importing data/disk images.
 Supported formats: qcow2, VMDK, VDI, VHD, VHDX, raw XZ-compressed, gzip-compressed, and uncompressed raw files can be imported.  
 They will all be converted to the raw format.
 
-Supported sources: http, https, http with basic auth, docker registry, S3 buckets, upload.
+Supported sources: http, https, http with basic auth, docker registry, S3 buckets, GCS Buckets, upload.
 
 Note: Some of these operations require [scratch space](scratch-space.md), doubling the storage space requirement of the import and the writes.  
 This is done with some misbehaving servers (not supporting HEAD requests), custom CAs, and during upload.

go.mod

@@ -3,6 +3,7 @@ module kubevirt.io/containerized-data-importer
 go 1.19
 
 require (
+	cloud.google.com/go/storage v1.29.0
 	github.com/appscode/jsonpatch v1.0.1
 	github.com/aws/aws-sdk-go v1.25.48
 	github.com/containers/image/v5 v5.19.1
@@ -39,6 +40,7 @@ require (
 	github.com/vmware/govmomi v0.23.1
 	go.uber.org/zap v1.19.1
 	golang.org/x/sys v0.5.0
+	google.golang.org/api v0.106.0
 	gopkg.in/fsnotify.v1 v1.4.7
 	gopkg.in/square/go-jose.v2 v2.5.1
 	k8s.io/api v0.25.0
@@ -62,6 +64,10 @@ require (
 )
 
 require (
+	cloud.google.com/go v0.107.0 // indirect
+	cloud.google.com/go/compute v1.14.0 // indirect
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	cloud.google.com/go/iam v0.8.0 // indirect
 	github.com/BurntSushi/toml v1.0.0 // indirect
 	github.com/Microsoft/go-winio v0.5.1 // indirect
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
@@ -88,8 +94,10 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect
+	github.com/googleapis/gax-go/v2 v2.7.0 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
@@ -119,19 +127,23 @@ require (
 	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
 	github.com/vbatts/tar-split v0.11.2 // indirect
 	go.etcd.io/bbolt v1.3.6 // indirect
+	go.opencensus.io v0.24.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
 	golang.org/x/net v0.7.0 // indirect
-	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
+	golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
 	golang.org/x/term v0.5.0 // indirect
 	golang.org/x/text v0.7.0 // indirect
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	golang.org/x/tools v0.1.12 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
+	google.golang.org/grpc v1.51.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect