storage: add s3 backend support (without GC and dedupe)

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>

.github/workflows/ci-cd.yml

@@ -13,6 +13,14 @@ jobs:
   build-test:
     name: Build and test ZOT
     runs-on: ubuntu-latest
+    services:
+      s3mock:
+        image: localstack/localstack-full
+        env:
+          SERVICES: s3
+        ports:
+          - 4563-4599:4563-4599
+          - 9090:8080
     steps:
       - name: Install go
         uses: actions/setup-go@v2
@@ -36,6 +44,10 @@ jobs:
         timeout-minutes: 30
         run: |
           cd $GITHUB_WORKSPACE && make
+        env:
+          S3MOCK_ENDPOINT: localhost:4566
+          AWS_ACCESS_KEY_ID: fake
+          AWS_SECRET_ACCESS_KEY: fake
 
       - name: Upload code coverage
         uses: codecov/codecov-action@v1  

examples/README.md

@@ -245,3 +245,16 @@ Enable audit logs and set output file with:
     "audit": "/tmp/zot-audit.log"
   }
 ```
+
+## Storage Drivers
+
+zot supports multiple storage options, including:
+- [s3](https://github.com/docker/docker.github.io/blob/master/registry/storage-drivers/s3.md): A driver storing objects in an Amazon Simple Storage Service (S3) bucket.
+- [azure](https://github.com/docker/docker.github.io/blob/master/registry/storage-drivers/azure.md): A driver storing objects in Microsoft Azure Blob Storage.
+- [swift](https://github.com/docker/docker.github.io/blob/master/registry/storage-drivers/swift.md): A driver storing objects in Openstack Swift.
+- [oss](https://github.com/docker/docker.github.io/blob/master/registry/storage-drivers/oss.md): A driver storing objects in Aliyun OSS.
+- [gcs](https://github.com/docker/docker.github.io/blob/master/registry/storage-drivers/gcs.md): A driver storing objects in a Google Cloud Storage bucket.
+
+For an s3 zot configuration with multiple storage drivers see: [s3-config](config-s3.json).
+zot also supports different storage drivers for each subpath.
+

examples/config-s3.json

@@ -0,0 +1,53 @@
+{
+    "version": "0.1.0-dev",
+    "storage": {
+        "rootDirectory": "/zot",
+        "storageDriver": {
+            "name": "s3",
+            "region": "us-east-2",
+            "bucket": "zot-storage",
+            "secure": true,
+            "skipverify": false
+        },
+        "subPaths": {
+            "/a": {
+                "rootDirectory": "/zot-a",
+                "storageDriver": {
+                    "name": "s3",
+                    "region": "us-east-2",
+                    "bucket": "zot-storage",
+                    "secure": true,
+                    "skipverify": false
+                }
+            },
+            "/b": {
+                "rootDirectory": "/zot-b",
+                "storageDriver": {
+                    "name": "s3",
+                    "region": "us-east-2",
+                    "bucket": "zot-storage",
+                    "secure": true,
+                    "skipverify": false
+                }
+            },
+            "/c": {
+                "rootDirectory": "/zot-c",
+                "storageDriver": {
+                    "name": "s3",
+                    "region": "us-east-2",
+                    "bucket": "zot-storage",
+                    "secure": false,
+                    "skipverify": false
+                }
+            }
+        }
+    },
+    "http": {
+        "address": "127.0.0.1",
+        "port": "8080",
+        "ReadOnly": false
+    },
+    "log": {
+        "level": "debug"
+    }
+}

go.mod

@@ -16,11 +16,13 @@ require (
 	github.com/containers/common v0.26.0
 	github.com/containers/image/v5 v5.13.2
 	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
+	github.com/docker/distribution v2.7.1+incompatible
 	github.com/dustin/go-humanize v1.0.0
 	github.com/fsnotify/fsnotify v1.5.1
 	github.com/getlantern/deepcopy v0.0.0-20160317154340-7f45deb8130a
 	github.com/go-ldap/ldap/v3 v3.4.1
 	github.com/gofrs/uuid v4.0.0+incompatible
+	github.com/golang/mock v1.6.0 // indirect
 	github.com/google/go-containerregistry v0.6.0
 	github.com/gorilla/handlers v1.5.1
 	github.com/gorilla/mux v1.8.0

go.sum

@@ -725,8 +725,9 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
 github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
-github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g=
 github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
+github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
+github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

pkg/api/config/config.go

@@ -20,6 +20,7 @@ type StorageConfig struct {
 	RootDirectory string
 	GC            bool
 	Dedupe        bool
+	StorageDriver map[string]interface{} `mapstructure:",omitempty"`
 }
 
 type TLSConfig struct {
@@ -80,6 +81,7 @@ type GlobalStorageConfig struct {
 	RootDirectory string
 	Dedupe        bool
 	GC            bool
+	StorageDriver map[string]interface{} `mapstructure:",omitempty"`
 	SubPaths      map[string]StorageConfig
 }
 

pkg/api/controller.go

@@ -14,8 +14,11 @@ import (
 	ext "github.com/anuvu/zot/pkg/extensions"
 	"github.com/anuvu/zot/pkg/log"
 	"github.com/anuvu/zot/pkg/storage"
+	"github.com/anuvu/zot/pkg/storage/s3"
 	"github.com/gorilla/handlers"
 	"github.com/gorilla/mux"
+
+	"github.com/docker/distribution/registry/storage/driver/factory"
 )
 
 const (
@@ -96,8 +99,23 @@ func (c *Controller) Run() error {
 			}
 		}
 
-		defaultStore := storage.NewImageStore(c.Config.Storage.RootDirectory,
-			c.Config.Storage.GC, c.Config.Storage.Dedupe, c.Log)
+		var defaultStore storage.ImageStore
+		if len(c.Config.Storage.StorageDriver) == 0 {
+			defaultStore = storage.NewImageStore(c.Config.Storage.RootDirectory,
+				c.Config.Storage.GC, c.Config.Storage.Dedupe, c.Log)
+		} else {
+			storeName := fmt.Sprintf("%v", c.Config.Storage.StorageDriver["name"])
+
+			// Init a Storager from connection string.
+			store, err := factory.Create(storeName, c.Config.Storage.StorageDriver)
+			if err != nil {
+				c.Log.Error().Err(err).Str("rootDir", c.Config.Storage.RootDirectory).Msg("Unable to create s3 service")
+				return err
+			}
+
+			defaultStore = s3.NewImageStore(c.Config.Storage.RootDirectory,
+				c.Config.Storage.GC, c.Config.Storage.Dedupe, c.Log, store)
+		}
 
 		c.StoreController.DefaultStore = defaultStore
 
@@ -130,8 +148,22 @@ func (c *Controller) Run() error {
 					}
 				}
 
-				subImageStore[route] = storage.NewImageStore(storageConfig.RootDirectory,
-					storageConfig.GC, storageConfig.Dedupe, c.Log)
+				if len(storageConfig.StorageDriver) == 0 {
+					subImageStore[route] = storage.NewImageStore(storageConfig.RootDirectory,
+						storageConfig.GC, storageConfig.Dedupe, c.Log)
+				} else {
+					storeName := fmt.Sprintf("%v", storageConfig.StorageDriver["name"])
+
+					// Init a Storager from connection string.
+					store, err := factory.Create(storeName, storageConfig.StorageDriver)
+					if err != nil {
+						c.Log.Error().Err(err).Str("rootDir", storageConfig.RootDirectory).Msg("Unable to create s3 service")
+						return err
+					}
+
+					subImageStore[route] = s3.NewImageStore(storageConfig.RootDirectory,
+						storageConfig.GC, storageConfig.Dedupe, c.Log, store)
+				}
 
 				// Enable extensions if extension config is provided
 				if c.Config != nil && c.Config.Extensions != nil {

pkg/api/controller_test.go

@@ -123,6 +123,12 @@ func getCredString(username, password string) string {
 	return usernameAndHash
 }
 
+func skipIt(t *testing.T) {
+	if os.Getenv("S3MOCK_ENDPOINT") == "" {
+		t.Skip("Skipping testing without AWS S3 mock server")
+	}
+}
+
 func TestNew(t *testing.T) {
 	Convey("Make a new controller", t, func() {
 		conf := config.New()
@@ -131,6 +137,126 @@ func TestNew(t *testing.T) {
 	})
 }
 
+func TestObjectStorageController(t *testing.T) {
+	skipIt(t)
+	Convey("Negative make a new object storage controller", t, func() {
+		port := getFreePort()
+		conf := config.New()
+		conf.HTTP.Port = port
+		storageDriverParams := map[string]interface{}{
+			"rootDir": "zot",
+			"name":    "s3",
+		}
+		conf.Storage.StorageDriver = storageDriverParams
+		c := api.NewController(conf)
+		So(c, ShouldNotBeNil)
+
+		c.Config.Storage.RootDirectory = "zot"
+
+		err := c.Run()
+		So(err, ShouldNotBeNil)
+	})
+
+	Convey("Make a new object storage controller", t, func() {
+		port := getFreePort()
+		baseURL := getBaseURL(port, false)
+		conf := config.New()
+		conf.HTTP.Port = port
+
+		bucket := "zot-storage-test"
+		endpoint := os.Getenv("S3MOCK_ENDPOINT")
+
+		storageDriverParams := map[string]interface{}{
+			"rootDir":        "zot",
+			"name":           "s3",
+			"region":         "us-east-2",
+			"bucket":         bucket,
+			"regionendpoint": endpoint,
+			"secure":         false,
+			"skipverify":     false,
+		}
+		conf.Storage.StorageDriver = storageDriverParams
+		c := api.NewController(conf)
+		So(c, ShouldNotBeNil)
+
+		c.Config.Storage.RootDirectory = "/"
+
+		go func(controller *api.Controller) {
+			// this blocks
+			if err := controller.Run(); err != nil {
+				return
+			}
+		}(c)
+
+		// wait till ready
+		for {
+			_, err := resty.R().Get(baseURL)
+			if err == nil {
+				break
+			}
+			time.Sleep(100 * time.Millisecond)
+		}
+
+		defer func(controller *api.Controller) {
+			ctx := context.Background()
+			_ = controller.Server.Shutdown(ctx)
+		}(c)
+	})
+}
+
+func TestObjectStorageControllerSubPaths(t *testing.T) {
+	skipIt(t)
+	Convey("Make a new object storage controller", t, func() {
+		port := getFreePort()
+		baseURL := getBaseURL(port, false)
+		conf := config.New()
+		conf.HTTP.Port = port
+
+		bucket := "zot-storage-test"
+		endpoint := os.Getenv("S3MOCK_ENDPOINT")
+
+		storageDriverParams := map[string]interface{}{
+			"rootDir":        "zot",
+			"name":           "s3",
+			"region":         "us-east-2",
+			"bucket":         bucket,
+			"regionendpoint": endpoint,
+			"secure":         false,
+			"skipverify":     false,
+		}
+		conf.Storage.StorageDriver = storageDriverParams
+		c := api.NewController(conf)
+		So(c, ShouldNotBeNil)
+
+		c.Config.Storage.RootDirectory = "zot"
+		subPathMap := make(map[string]config.StorageConfig)
+		subPathMap["/a"] = config.StorageConfig{
+			RootDirectory: "/a",
+			StorageDriver: storageDriverParams,
+		}
+		c.Config.Storage.SubPaths = subPathMap
+
+		go func(controller *api.Controller) {
+			// this blocks
+			if err := controller.Run(); err != nil {
+				return
+			}
+		}(c)
+
+		for {
+			_, err := resty.R().Get(baseURL)
+			if err == nil {
+				break
+			}
+			time.Sleep(100 * time.Millisecond)
+		}
+		defer func(controller *api.Controller) {
+			ctx := context.Background()
+			_ = controller.Server.Shutdown(ctx)
+		}(c)
+	})
+}
+
 func TestHtpasswdSingleCred(t *testing.T) {
 	Convey("Single cred", t, func() {
 		port := getFreePort()