From 9f343a9ceb299892ac1126da4a6db4c61be45a51 Mon Sep 17 00:00:00 2001 From: fedor Date: Tue, 17 Apr 2018 08:07:00 -0400 Subject: [PATCH] GCS Proxy --- .cirrus.yml | 9 ++++ .gitignore | 4 ++ README.md | 4 +- cmd/main.go | 36 +++++++++++++ proxy/http_proxy.go | 109 +++++++++++++++++++++++++++++++++++++++ proxy/http_proxy_test.go | 101 ++++++++++++++++++++++++++++++++++++ 6 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 .cirrus.yml create mode 100644 cmd/main.go create mode 100644 proxy/http_proxy.go create mode 100644 proxy/http_proxy_test.go diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 0000000..e27ee28 --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,9 @@ +container: + image: golang:1.10 + +test_task: + env: + CIRRUS_WORKING_DIR: /go/src/github.com/$CIRRUS_REPO_FULL_NAME + get_script: go get -t -v ./... + vet_script: go vet -v ./... + test_script: go test -v ./... \ No newline at end of file diff --git a/.gitignore b/.gitignore index a1338d6..eac7e5f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,7 @@ # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 .glide/ + +# IntelliJ +.idea/ +*.iml diff --git a/README.md b/README.md index 9a095c9..ca5ea26 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,4 @@ -# google-storage-proxy HTTP proxy with REST API to interact with Google Cloud Storage Buckets + +Simply allows to use `HEAD`, `GET` or `PUT` requests to check blob's availability, as well as downloading or uploading +blobs to a specified GCS bucket. diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 0000000..ed0c602 --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,36 @@ +package main + +import ( + "cloud.google.com/go/storage" + "context" + "flag" + "github.com/cirruslabs/google-storage-proxy/proxy" + "log" +) + +func main() { + var port int64 + flag.Int64Var(&port, "port", 8080, "Port to serve") + var bucketName string + flag.StringVar(&bucketName, "bucket", "", "Google Storage Bucket Name") + var defaultPrefix string + flag.StringVar(&defaultPrefix, "prefix", "", "Default prefix for all object names. For example, use --prefix=foo/.") + flag.Parse() + + if bucketName == "" { + log.Fatal("Please specify Google Cloud Storage Bucket") + } + + client, err := storage.NewClient(context.Background()) + if err != nil { + log.Fatalf("Failed to create a storage client: %s", err) + } + + bucketHandler := client.Bucket(bucketName) + storageProxy := http_cache.NewStorageProxy(bucketHandler, defaultPrefix) + + err = storageProxy.Serve(port) + if err != nil { + log.Fatalf("Failed to start proxy: %s", err) + } +} diff --git a/proxy/http_proxy.go b/proxy/http_proxy.go new file mode 100644 index 0000000..a8ccd07 --- /dev/null +++ b/proxy/http_proxy.go @@ -0,0 +1,109 @@ +package http_cache + +import ( + "bufio" + "context" + "fmt" + "cloud.google.com/go/storage" + "log" + "net" + "net/http" +) + +type StorageProxy struct { + bucketHandler *storage.BucketHandle + defaultPrefix string +} + +func NewStorageProxy(bucketHandler *storage.BucketHandle, defaultPrefix string) *StorageProxy { + return &StorageProxy{ + bucketHandler: bucketHandler, + defaultPrefix: defaultPrefix, + } +} + +func (proxy StorageProxy) objectName(name string) string { + return proxy.defaultPrefix + name +} + +func (proxy StorageProxy) Serve(port int64) error { + http.HandleFunc("/", proxy.handler) + + listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) + + if err == nil { + address := listener.Addr().String() + listener.Close() + log.Printf("Starting http cache server %s\n", address) + return http.ListenAndServe(address, nil) + } + return err +} + +func (proxy StorageProxy) handler(w http.ResponseWriter, r *http.Request) { + key := r.URL.Path + if key[0] == '/' { + key = key[1:] + } + if r.Method == "GET" { + proxy.downloadBlob(w, key) + } else if r.Method == "HEAD" { + proxy.checkBlobExists(w, key) + } else if r.Method == "POST" { + proxy.uploadBlob(w, r, key) + } else if r.Method == "PUT" { + proxy.uploadBlob(w, r, key) + } +} + +func (proxy StorageProxy) downloadBlob(w http.ResponseWriter, name string) { + object := proxy.bucketHandler.Object(proxy.objectName(name)) + if object == nil { + w.WriteHeader(http.StatusNotFound) + return + } + reader, err := object.NewReader(context.Background()) + if err != nil { + w.WriteHeader(http.StatusNotFound) + return + } + defer reader.Close() + bufferedReader := bufio.NewReader(reader) + _, err = bufferedReader.WriteTo(w) + if err != nil { + w.WriteHeader(http.StatusNotFound) + return + } + w.WriteHeader(http.StatusOK) +} + +func (proxy StorageProxy) checkBlobExists(w http.ResponseWriter, name string) { + object := proxy.bucketHandler.Object(proxy.objectName(name)) + if object == nil { + w.WriteHeader(http.StatusNotFound) + return + } + // lookup attributes to see if the object exists + attrs, err := object.Attrs(context.Background()) + if err != nil || attrs == nil { + w.WriteHeader(http.StatusNotFound) + return + } + w.WriteHeader(http.StatusOK) +} + +func (proxy StorageProxy) uploadBlob(w http.ResponseWriter, r *http.Request, name string) { + object := proxy.bucketHandler.Object(proxy.objectName(name)) + + writer := object.NewWriter(context.Background()) + defer writer.Close() + + _, err := bufio.NewWriter(writer).ReadFrom(bufio.NewReader(r.Body)) + if err != nil { + errorMsg := fmt.Sprintf("Failed read cache body! %s", err) + w.Write([]byte(errorMsg)) + w.WriteHeader(http.StatusBadRequest) + return + } + w.WriteHeader(http.StatusCreated) +} diff --git a/proxy/http_proxy_test.go b/proxy/http_proxy_test.go new file mode 100644 index 0000000..898bd95 --- /dev/null +++ b/proxy/http_proxy_test.go @@ -0,0 +1,101 @@ +package http_cache + +import ( + "testing" + "github.com/fsouza/fake-gcs-server/fakestorage" + "net/http/httptest" + "net/http" + "strings" +) + +const TestBucketName = "some-bucket" + +func Test_Blob_Exists(t *testing.T) { + server := fakestorage.NewServer([]fakestorage.Object{ + { + BucketName: TestBucketName, + Name: "some/object/file", + }, + }) + defer server.Stop() + client := server.Client() + storageProxy := NewStorageProxy(client.Bucket(TestBucketName), "") + + response := httptest.NewRecorder() + storageProxy.checkBlobExists(response, "some/object/file") + + if response.Code == http.StatusOK { + t.Log("Passed") + } else { + t.Errorf("Wrong status: '%d'", response.Code) + } +} + +func Test_Default_Prefix(t *testing.T) { + server := fakestorage.NewServer([]fakestorage.Object{ + { + BucketName: TestBucketName, + Name: "some/object/file", + }, + }) + defer server.Stop() + client := server.Client() + storageProxy := NewStorageProxy(client.Bucket(TestBucketName), "some/object/") + + response := httptest.NewRecorder() + storageProxy.checkBlobExists(response, "file") + + if response.Code == http.StatusOK { + t.Log("Passed") + } else { + t.Errorf("Wrong status: '%d'", response.Code) + } +} + +func Test_Blob_Download(t *testing.T) { + expectedBlobContent := "my content" + server := fakestorage.NewServer([]fakestorage.Object{ + { + BucketName: TestBucketName, + Name: "some/file", + Content: []byte(expectedBlobContent), + }, + }) + defer server.Stop() + client := server.Client() + storageProxy := NewStorageProxy(client.Bucket(TestBucketName), "") + + response := httptest.NewRecorder() + storageProxy.downloadBlob(response, "some/file") + + if response.Code == http.StatusOK { + t.Log("Passed") + } else { + t.Errorf("Wrong status: '%d'", response.Code) + } + + downloadedBlobContent := response.Body.String() + if downloadedBlobContent == expectedBlobContent { + t.Log("Passed") + } else { + t.Errorf("Wrong content: '%s'", downloadedBlobContent) + } +} + +func Test_Blob_Upload(t *testing.T) { + expectedBlobContent := "my content" + server := fakestorage.NewServer([]fakestorage.Object{}) + defer server.Stop() + client := server.Client() + storageProxy := NewStorageProxy(client.Bucket(TestBucketName), "") + + response := httptest.NewRecorder() + request := httptest.NewRequest("POST", "/test-file", strings.NewReader(expectedBlobContent)) + storageProxy.uploadBlob(response, request,"test-file") + + if response.Code == http.StatusCreated { + t.Log("Passed") + } else { + t.Errorf("Wrong status: '%d'", response.Code) + } +} \ No newline at end of file