test: GCS fs

fs/fs_test.go

@@ -4,15 +4,18 @@ import (
 	gocontext "context"
 	"errors"
 	"flag"
+	"io"
 	"os"
 	"strings"
 	"testing"
 	"time"
 
+	"cloud.google.com/go/storage"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/flanksource/duty/connection"
 	"github.com/flanksource/duty/context"
 	"github.com/flanksource/duty/types"
+	"google.golang.org/api/googleapi"
 
 	s3Types "github.com/aws/aws-sdk-go-v2/service/s3/types"
 )
@@ -33,6 +36,7 @@ func TestMain(m *testing.M) {
 	flag.Parse()
 	os.Exit(m.Run())
 }
+
 func TestFS(t *testing.T) {
 	parentCtx, cancel := gocontext.WithTimeout(gocontext.Background(), time.Minute)
 	defer cancel()
@@ -45,7 +49,12 @@ func TestFS(t *testing.T) {
 				t.Fatalf("%v", err)
 			}
 
-			files, err := client.fs.ReadDir("*")
+			dir := "*"
+			if client.name == "gcsFS" {
+				dir = ""
+			}
+
+			files, err := client.fs.ReadDir(dir)
 			if err != nil {
 				t.Fatalf("%v", err)
 			}
@@ -54,6 +63,28 @@ func TestFS(t *testing.T) {
 				t.Fatalf("expected 5 files, got %d", len(files))
 			}
 
+			// GCS doesn't support glob yet
+			if client.name == "gcsFS" {
+				return
+			}
+
+			{
+				file, err := client.fs.Read(ctx, "record-1.txt")
+				if err != nil {
+					t.Fatalf("%v", err)
+				}
+				defer file.Close()
+
+				content, err := io.ReadAll(file)
+				if err != nil {
+					t.Fatalf("%v", err)
+				}
+
+				if string(content) != "record-1" {
+					t.Errorf("expected content to be record-1, got %s", string(content))
+				}
+			}
+
 			{
 				files, err := client.fs.ReadDir("*.json")
 				if err != nil {
@@ -122,6 +153,17 @@ type testData struct {
 func getTestClients(t *testing.T, ctx context.Context) []testData {
 	t.Helper()
 
+	gcsFS, err := NewGCSFS(ctx, "test", connection.GCSConnection{
+		GCPConnection: connection.GCPConnection{
+			SkipTLSVerify: true,
+			Endpoint:      "https://localhost:4443/storage/v1/",
+		},
+	})
+	if err != nil {
+		t.Fatalf("failed to create GCS filesystem: %v", err)
+	}
+	createGSCBucket(t, ctx, gcsFS.Client, "fkae-project", "test")
+
 	sshfs, err := NewSSHFS("localhost:2222", "foo", "pass")
 	if err != nil {
 		t.Fatalf("%v", err)
@@ -151,6 +193,7 @@ func getTestClients(t *testing.T, ctx context.Context) []testData {
 	createBucket(t, s3FS.Client, *bucket)
 
 	testClients := []testData{
+		{"gcsFS", gcsFS},
 		{"sshfs", sshfs},
 		{"smbfs", smbFS},
 		{"s3FS", s3FS},
@@ -175,6 +218,22 @@ func createBucket(t *testing.T, cl *s3.Client, bucket string) {
 	}
 }
 
+func createGSCBucket(t *testing.T, ctx context.Context, cl *storage.Client, projectID, bucket string) {
+	t.Helper()
+
+	err := cl.Bucket(bucket).Create(ctx, projectID, nil)
+	if err != nil {
+		var gErr *googleapi.Error
+		if errors.As(err, &gErr) {
+			if gErr.Code == 409 {
+				return
+			}
+		}
+
+		t.Fatal(err)
+	}
+}
+
 func envDefault(env, def string) string {
 	if os.Getenv(env) == "" {
 		return def

fs/gcs.go

@@ -21,17 +21,17 @@ type gcsFS struct {
 }
 
 func NewGCSFS(ctx context.Context, bucket string, conn connection.GCSConnection) (*gcsFS, error) {
-	cfg, err := gcpUtil.NewSession(ctx, &conn.GCPConnection)
+	client, err := conn.GCPConnection.Client(ctx)
 	if err != nil {
 		return nil, err
 	}
 
-	client := gcsFS{
+	fs := gcsFS{
 		Bucket: strings.TrimPrefix(bucket, "gcs://"),
-		Client: cfg,
+		Client: client,
 	}
 
-	return &client, nil
+	return &fs, nil
 }
 
 func (t *gcsFS) Close() error {

fs/testdata/docker-compose.yml

@@ -1,15 +1,9 @@
 services:
-  localstack:
-    container_name: 'localstack'
-    image: localstack/localstack:0.14.0
-    network_mode: bridge
+  gcs:
+    image: fsouza/fake-gcs-server
+    container_name: fake-gcs-server
     ports:
-      - '4566:4566'
-      - '4571:4571'
-    environment:
-      - SERVICES=s3
-    profiles:
-      - disable
+      - 4443:4443
 
   minio:
     image: minio/minio