Skip to content

Commit

Permalink
Google Cloud Storage Datasource - #4
Browse files Browse the repository at this point in the history
Fixed some linting issues.

Signed-off-by: Marcelo Feitoza Parisi <marcelo@feitoza.com.br>
  • Loading branch information
feitnomore committed Mar 6, 2023
1 parent 6b3e6db commit c75ba2e
Show file tree
Hide file tree
Showing 2 changed files with 31 additions and 33 deletions.
62 changes: 30 additions & 32 deletions pkg/importer/gcs-datasource.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,15 +65,15 @@ func NewGCSDataSource(endpoint, keyFile string) (*GCSDataSource, error) {
bucket, object := extractGcsBucketAndObject(endpoint)

// Creating GCS Client
client, err := getGcsClientGs(keyFile, ctx)
client, err := getGcsClientGs(ctx, keyFile)
if err != nil {
klog.Errorf("GCS Importer: Error creating GS Client")
return nil, err
}
defer client.Close()

// Creating GCS Reader
gcsReader, err := newReaderFunc(client, bucket, object, ctx)
gcsReader, err := newReaderFunc(ctx, client, bucket, object)
if err != nil {
klog.Errorf("GCS Importer: Error creating Reader")
return nil, err
Expand All @@ -84,30 +84,30 @@ func NewGCSDataSource(endpoint, keyFile string) (*GCSDataSource, error) {
gcsReader: gcsReader,
}, nil

} else { // Using http(s):// endpoint
// Extracting bucket, object name and host
bucket, object, host := extractGcsBucketObjectAndHost(endpoint)
}

// Creating GCS Client
client, err := getGcsClientHttp(keyFile, ctx, host)
if err != nil {
klog.Errorf("GCS Importer: Error creating HTTP(s) Client")
return nil, err
}
defer client.Close()
// Using http(s):// endpoint and extracting bucket, object name and host
bucket, object, host := extractGcsBucketObjectAndHost(endpoint)

// Creating GCS Reader
gcsReader, err := newReaderFunc(client, bucket, object, ctx)
if err != nil {
klog.Errorf("GCS Importer: Error creating Reader")
return nil, err
}
return &GCSDataSource{
ep: ep,
keyFile: keyFile,
gcsReader: gcsReader,
}, nil
// Creating GCS Client
client, err := getGcsClientHTTP(ctx, keyFile, host)
if err != nil {
klog.Errorf("GCS Importer: Error creating HTTP(s) Client")
return nil, err
}
defer client.Close()

// Creating GCS Reader
gcsReader, err := newReaderFunc(ctx, client, bucket, object)
if err != nil {
klog.Errorf("GCS Importer: Error creating Reader")
return nil, err
}
return &GCSDataSource{
ep: ep,
keyFile: keyFile,
gcsReader: gcsReader,
}, nil

}

Expand Down Expand Up @@ -175,32 +175,30 @@ func (sd *GCSDataSource) Close() error {
}

// Create a Cloud Storage Client
func getGcsClientGs(keyFile string, ctx context.Context) (*storage.Client, error) {
func getGcsClientGs(ctx context.Context, keyFile string) (*storage.Client, error) {
klog.V(3).Infoln("GCS Importer: Creating Client")
if keyFile != "" {
klog.V(3).Infoln("GCS Importer: Creating Client: Authenticated")
return storage.NewClient(ctx)
} else {
klog.V(3).Infoln("GCS Importer: Creating Client: Anonymous")
return storage.NewClient(ctx, option.WithoutAuthentication())
}
klog.V(3).Infoln("GCS Importer: Creating Client: Anonymous")
return storage.NewClient(ctx, option.WithoutAuthentication())
}

// Create a Cloud Storage Client
func getGcsClientHttp(keyFile string, ctx context.Context, host string) (*storage.Client, error) {
func getGcsClientHTTP(ctx context.Context, keyFile string, host string) (*storage.Client, error) {
klog.V(3).Infoln("GCS Importer: Creating HTTP(s) Client")
klog.V(3).Infoln("GCS Importer: HTTP(s) Host:", host)
if keyFile != "" {
klog.V(3).Infoln("GCS Importer: Creating Client HTTP(s): Authenticated")
return storage.NewClient(ctx, option.WithEndpoint(host))
} else {
klog.V(3).Infoln("GCS Importer: Creating Client HTTP(s): Anonymous")
return storage.NewClient(ctx, option.WithoutAuthentication(), option.WithEndpoint(host))
}
klog.V(3).Infoln("GCS Importer: Creating Client HTTP(s): Anonymous")
return storage.NewClient(ctx, option.WithoutAuthentication(), option.WithEndpoint(host))
}

// Create Cloud Storage Object Reader
func getGcsObjectReader(client *storage.Client, bucket, object string, ctx context.Context) (io.ReadCloser, error) {
func getGcsObjectReader(ctx context.Context, client *storage.Client, bucket, object string) (io.ReadCloser, error) {
klog.V(3).Infoln("GCS Importer: Creating Reader for bucket:", bucket, "object:", object)
return client.Bucket(bucket).Object(object).NewReader(ctx)
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/importer/gcs-datasource_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -476,7 +476,7 @@ var _ = Describe("Google Cloud Storage data source", func() {
})

// Create Cloud Storage Object Reader pointing to a sample image
func mockGcsObjectReader(client *storage.Client, bucket, object string, ctx context.Context) (io.ReadCloser, error) {
func mockGcsObjectReader(ctx context.Context, client *storage.Client, bucket, object string) (io.ReadCloser, error) {
var sampleImage = filepath.Join(imageDir, "cirros.raw")
return os.Open(sampleImage)
}

0 comments on commit c75ba2e

Please sign in to comment.