Skip to content

Commit

Permalink
Google Cloud Storage Datasource - #3
Browse files Browse the repository at this point in the history 
Added functional test coverage for GCS Importer.
Added a rewrite to file-host to support GCS tests.

Signed-off-by: Marcelo Feitoza Parisi <marcelo@feitoza.com.br>
  • Loading branch information
@feitnomore
feitnomore committed Mar 2, 2023
1 parent df8b7b4 commit 38d8e43
Show file tree
Hide file tree
Showing 6 changed files with 96 additions and 12 deletions.
Empty file removed 0
View file
Empty file.
1 change: 1 addition & 0 deletions pkg/importer/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ go_test(
"//pkg/util/cert/triple:go_default_library",
"//tests/reporters:go_default_library",
"//tests/utils:go_default_library",
"//vendor/cloud.google.com/go/storage:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/service/s3:go_default_library",
"//vendor/github.com/onsi/ginkgo:go_default_library",
"//vendor/github.com/onsi/ginkgo/extensions/table:go_default_library",
Expand Down
19 changes: 7 additions & 12 deletions pkg/importer/gcs-datasource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"net/url"
"path/filepath"
"strings"
"time"

"github.com/pkg/errors"

Expand All @@ -22,7 +21,6 @@ import (
const (
gcsFolderSep = "/"
gcsScheme = "gs"
retrySeconds = 60
)

// Helper for unit-testing
Expand Down Expand Up @@ -60,25 +58,22 @@ func NewGCSDataSource(endpoint, keyFile string) (*GCSDataSource, error) {

// Getting Context
ctx := context.Background()
klog.V(3).Infoln("GCS Importer: Timeout in seconds:", retrySeconds)
tctx, cancel := context.WithTimeout(ctx, retrySeconds*time.Second)
defer cancel() // Cancel when done, whether we time out or not.

// Using gs:// endpoint
if strings.HasPrefix(endpoint, "gs") {
// Extracting bucket and object name
bucket, object := extractGcsBucketAndObject(endpoint)

// Creating GCS Client
client, err := getGcsClientGs(keyFile, tctx)
client, err := getGcsClientGs(keyFile, ctx)
if err != nil {
klog.Errorf("GCS Importer: Error creating GS Client")
return nil, err
}
defer client.Close()

// Creating GCS Reader
gcsReader, err := newReaderFunc(client, bucket, object, tctx)
gcsReader, err := newReaderFunc(client, bucket, object, ctx)
if err != nil {
klog.Errorf("GCS Importer: Error creating Reader")
return nil, err
Expand All @@ -94,15 +89,15 @@ func NewGCSDataSource(endpoint, keyFile string) (*GCSDataSource, error) {
bucket, object, host := extractGcsBucketObjectAndHost(endpoint)

// Creating GCS Client
client, err := getGcsClientHttp(keyFile, tctx, host)
client, err := getGcsClientHttp(keyFile, ctx, host)
if err != nil {
klog.Errorf("GCS Importer: Error creating HTTP(s) Client")
return nil, err
}
defer client.Close()

// Creating GCS Reader
gcsReader, err := newReaderFunc(client, bucket, object, tctx)
gcsReader, err := newReaderFunc(client, bucket, object, ctx)
if err != nil {
klog.Errorf("GCS Importer: Error creating Reader")
return nil, err
Expand Down Expand Up @@ -193,14 +188,14 @@ func getGcsClientGs(keyFile string, ctx context.Context) (*storage.Client, error

// Create a Cloud Storage Client
func getGcsClientHttp(keyFile string, ctx context.Context, host string) (*storage.Client, error) {
newEp := host + "/storage/v1/"
klog.V(3).Infoln("GCS Importer: Creating HTTP(s) Client")
klog.V(3).Infoln("GCS Importer: HTTP(s) Host:", host)
if keyFile != "" {
klog.V(3).Infoln("GCS Importer: Creating Client HTTP(s): Authenticated")
return storage.NewClient(ctx, option.WithEndpoint(newEp))
return storage.NewClient(ctx, option.WithEndpoint(host))
} else {
klog.V(3).Infoln("GCS Importer: Creating Client HTTP(s): Anonymous")
return storage.NewClient(ctx, option.WithoutAuthentication(), option.WithEndpoint(newEp))
return storage.NewClient(ctx, option.WithoutAuthentication(), option.WithEndpoint(host))
}
}

Expand Down
54 changes: 54 additions & 0 deletions tests/datavolume_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,12 @@ var _ = Describe("[vendor:cnv-qe@redhat.com][level:component]DataVolume tests",
cirrosURL := func() string {
return fmt.Sprintf(utils.CirrosURL, f.CdiInstallNs)
}
cirrosGCSQCOWURL := func() string {
return fmt.Sprintf(utils.CirrosGCSQCOWURL, f.CdiInstallNs)
}
cirrosGCSRAWURL := func() string {
return fmt.Sprintf(utils.CirrosGCSRAWURL, f.CdiInstallNs)
}
imageioURL := func() string {
return fmt.Sprintf(utils.ImageioURL, f.CdiInstallNs)
}
Expand Down Expand Up @@ -961,6 +967,54 @@ var _ = Describe("[vendor:cnv-qe@redhat.com][level:component]DataVolume tests",
Message: "Import Complete; VDDK: {\"Version\":\"1.2.3\",\"Host\":\"esx.test\"}",
Reason: "Completed",
}}),
table.Entry("[rfe_id:XXXX][crit:high][test_id:XXXX]succeed creating import dv from GCS URL using RAW image", dataVolumeTestArguments{
name: "dv-gcs-raw-import",
size: "1Gi",
url: cirrosGCSRAWURL,
dvFunc: utils.NewDataVolumeWithGCSImport,
eventReason: dvc.ImportSucceeded,
phase: cdiv1.Succeeded,
checkPermissions: true,
readyCondition: &cdiv1.DataVolumeCondition{
Type: cdiv1.DataVolumeReady,
Status: v1.ConditionTrue,
},
boundCondition: &cdiv1.DataVolumeCondition{
Type: cdiv1.DataVolumeBound,
Status: v1.ConditionTrue,
Message: "PVC dv-gcs-raw-import Bound",
Reason: "Bound",
},
runningCondition: &cdiv1.DataVolumeCondition{
Type: cdiv1.DataVolumeRunning,
Status: v1.ConditionFalse,
Message: "Import Complete",
Reason: "Completed",
}}),
table.Entry("[rfe_id:XXXX][crit:high][test_id:XXXX]succeed creating import dv from GCS URL using QCOW2 image", dataVolumeTestArguments{
name: "dv-gcs-qcow-import",
size: "1Gi",
url: cirrosGCSQCOWURL,
dvFunc: utils.NewDataVolumeWithGCSImport,
eventReason: dvc.ImportSucceeded,
phase: cdiv1.Succeeded,
checkPermissions: true,
readyCondition: &cdiv1.DataVolumeCondition{
Type: cdiv1.DataVolumeReady,
Status: v1.ConditionTrue,
},
boundCondition: &cdiv1.DataVolumeCondition{
Type: cdiv1.DataVolumeBound,
Status: v1.ConditionTrue,
Message: "PVC dv-gcs-qcow-import Bound",
Reason: "Bound",
},
runningCondition: &cdiv1.DataVolumeCondition{
Type: cdiv1.DataVolumeRunning,
Status: v1.ConditionFalse,
Message: "Import Complete",
Reason: "Completed",
}}),
)

savedVddkConfigMap := common.VddkConfigMap + "-saved"
Expand Down
31 changes: 31 additions & 0 deletions tests/utils/datavolume.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,10 @@ const (
TarArchiveURL = "http://cdi-file-host.%s/archive.tar"
// CirrosURL provides the standard cirros image qcow image
CirrosURL = "http://cdi-file-host.%s/cirros-qcow2.img"
// CirrosGCSQCOWURL provides the standard cirros image qcow image for GCS
CirrosGCSQCOWURL = "http://cdi-file-host.%s/gcs-bucket/cirros-qcow2.img"
// CirrosGCSRAWURL provides the standard cirros image raw image for GCS
CirrosGCSRAWURL = "http://cdi-file-host.%s/gcs-bucket/cirros.raw"
// ImageioURL provides URL of oVirt engine hosting imageio
ImageioURL = "https://imageio.%s:12346/ovirt-engine/api"
// ImageioRootURL provides the base path to fakeovirt, for inventory modifications
Expand Down Expand Up @@ -789,6 +793,33 @@ func NewDataVolumeWithVddkWarmImport(dataVolumeName string, size string, backing
}
}

// NewDataVolumeWithGCSImport initializes a DataVolume struct with GCS HTTP annotations
func NewDataVolumeWithGCSImport(dataVolumeName string, size string, gcsURL string) *cdiv1.DataVolume {
claimSpec := &k8sv1.PersistentVolumeClaimSpec{
AccessModes: []k8sv1.PersistentVolumeAccessMode{k8sv1.ReadWriteOnce},
Resources: k8sv1.ResourceRequirements{
Requests: k8sv1.ResourceList{
k8sv1.ResourceStorage: resource.MustParse(size),
},
},
}

return &cdiv1.DataVolume{
ObjectMeta: metav1.ObjectMeta{
Name: dataVolumeName,
Annotations: map[string]string{},
},
Spec: cdiv1.DataVolumeSpec{
Source: &cdiv1.DataVolumeSource{
GCS: &cdiv1.DataVolumeSourceGCS{
URL: gcsURL,
},
},
PVC: claimSpec,
},
}
}

// WaitForDataVolumePhase waits for DV to be in a specific phase (Pending, Bound, Succeeded etc.)
// or garbage collected if the passed in phase is Succeeded
func WaitForDataVolumePhase(ci ClientsIface, namespace string, phase cdiv1.DataVolumePhase, dataVolumeName string) error {
Expand Down
3 changes: 3 additions & 0 deletions tools/cdi-func-test-file-host-init/nginx.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ http {
location / {
autoindex on;
autoindex_format json;
rewrite ^/gcs-bucket/(.*)$ /$1 last;
}
}
# auth
Expand Down Expand Up @@ -61,6 +62,7 @@ http {
autoindex on;
autoindex_format json;
limit_rate 100k;
rewrite ^/gcs-bucket/(.*)$ /$1 last;
}
}
# tls
Expand All @@ -80,6 +82,7 @@ http {
location / {
autoindex on;
autoindex_format json;
rewrite ^/gcs-bucket/(.*)$ /$1 last;
}
}

Expand Down

0 comments on commit 38d8e43

Please sign in to comment.