storage: add s3 backend support (without GC and dedupe)

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>

.github/workflows/ci-cd.yml

@@ -13,6 +13,14 @@ jobs:
   build-test:
     name: Build and test ZOT
     runs-on: ubuntu-latest
+    services:
+      s3mock:
+        image: localstack/localstack-full
+        env:
+          SERVICES: s3
+        ports:
+          - 4563-4599:4563-4599
+          - 9090:8080
     steps:
       - name: Install go
         uses: actions/setup-go@v2
@@ -36,6 +44,10 @@ jobs:
         timeout-minutes: 30
         run: |
           cd $GITHUB_WORKSPACE && make
+        env:
+          S3MOCK_ENDPOINT: localhost:4566
+          AWS_ACCESS_KEY_ID: fake
+          AWS_SECRET_ACCESS_KEY: fake
 
       - name: Upload code coverage
         uses: codecov/codecov-action@v1  

examples/README.md

@@ -245,3 +245,72 @@ Enable audit logs and set output file with:
     "audit": "/tmp/zot-audit.log"
   }
 ```
+
+## Storage Drivers
+
+Beside filesystem storage backend, zot also supports S3 storage backend, check below url to see how to configure it:
+- [s3](https://github.com/docker/docker.github.io/blob/master/registry/storage-drivers/s3.md): A driver storing objects in an Amazon Simple Storage Service (S3) bucket.
+
+For an s3 zot configuration with multiple storage drivers see: [s3-config](config-s3.json).
+
+zot also supports different storage drivers for each subpath.
+
+### Specifying S3 credentials
+
+There are multiple ways to specify S3 credentials:
+
+- Config file: 
+
+```
+        "storageDriver": {
+            "name": "s3",
+            "region": "us-east-2",
+            "bucket": "zot-storage",
+            "secure": true,
+            "skipverify": false,
+            "accesskey": "<YOUR_ACCESS_KEY_ID>",
+            "secretkey": "<YOUR_SECRET_ACCESS_KEY>"
+        }
+```
+
+- Environment variables:
+
+SDK looks for credentials in the following environment variables:
+
+```
+    AWS_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY
+    AWS_SESSION_TOKEN (optional)
+```
+
+- Credentials file:
+
+A credential file is a plaintext file that contains your access keys. The file must be on the same machine on which you’re running your application. The file must be named credentials and located in the .aws/ folder in your home directory.
+
+```
+    [default]
+    aws_access_key_id = <YOUR_DEFAULT_ACCESS_KEY_ID>
+    aws_secret_access_key = <YOUR_DEFAULT_SECRET_ACCESS_KEY>
+
+    [test-account]
+    aws_access_key_id = <YOUR_TEST_ACCESS_KEY_ID>
+    aws_secret_access_key = <YOUR_TEST_SECRET_ACCESS_KEY>
+
+    [prod-account]
+    ; work profile
+    aws_access_key_id = <YOUR_PROD_ACCESS_KEY_ID>
+    aws_secret_access_key = <YOUR_PROD_SECRET_ACCESS_KEY>
+```
+
+The [default] heading defines credentials for the default profile, which the SDK will use unless you configure it to use another profile.
+
+To specify a profile use AWS_PROFILE environment variable:
+
+```
+AWS_PROFILE=test-account
+```
+
+For more details see https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials
+
+
+

examples/config-s3.json

@@ -0,0 +1,53 @@
+{
+    "version": "0.1.0-dev",
+    "storage": {
+        "rootDirectory": "/zot",
+        "storageDriver": {
+            "name": "s3",
+            "region": "us-east-2",
+            "bucket": "zot-storage",
+            "secure": true,
+            "skipverify": false
+        },
+        "subPaths": {
+            "/a": {
+                "rootDirectory": "/zot-a",
+                "storageDriver": {
+                    "name": "s3",
+                    "region": "us-east-2",
+                    "bucket": "zot-storage",
+                    "secure": true,
+                    "skipverify": false
+                }
+            },
+            "/b": {
+                "rootDirectory": "/zot-b",
+                "storageDriver": {
+                    "name": "s3",
+                    "region": "us-east-2",
+                    "bucket": "zot-storage",
+                    "secure": true,
+                    "skipverify": false
+                }
+            },
+            "/c": {
+                "rootDirectory": "/zot-c",
+                "storageDriver": {
+                    "name": "s3",
+                    "region": "us-east-2",
+                    "bucket": "zot-storage",
+                    "secure": false,
+                    "skipverify": false
+                }
+            }
+        }
+    },
+    "http": {
+        "address": "127.0.0.1",
+        "port": "8080",
+        "ReadOnly": false
+    },
+    "log": {
+        "level": "debug"
+    }
+}

go.mod

@@ -16,11 +16,13 @@ require (
 	github.com/containers/common v0.26.0
 	github.com/containers/image/v5 v5.13.2
 	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
+	github.com/docker/distribution v2.7.1+incompatible
 	github.com/dustin/go-humanize v1.0.0
 	github.com/fsnotify/fsnotify v1.5.1
 	github.com/getlantern/deepcopy v0.0.0-20160317154340-7f45deb8130a
 	github.com/go-ldap/ldap/v3 v3.4.1
 	github.com/gofrs/uuid v4.0.0+incompatible
+	github.com/golang/mock v1.6.0 // indirect
 	github.com/google/go-containerregistry v0.6.0
 	github.com/gorilla/handlers v1.5.1
 	github.com/gorilla/mux v1.8.0

go.sum

@@ -725,8 +725,9 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
 github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
-github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g=
 github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
+github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
+github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

pkg/api/config/config.go

@@ -20,6 +20,7 @@ type StorageConfig struct {
 	RootDirectory string
 	GC            bool
 	Dedupe        bool
+	StorageDriver map[string]interface{} `mapstructure:",omitempty"`
 }
 
 type TLSConfig struct {
@@ -80,6 +81,7 @@ type GlobalStorageConfig struct {
 	RootDirectory string
 	Dedupe        bool
 	GC            bool
+	StorageDriver map[string]interface{} `mapstructure:",omitempty"`
 	SubPaths      map[string]StorageConfig
 }
 

pkg/api/controller.go

@@ -14,8 +14,11 @@ import (
 	ext "github.com/anuvu/zot/pkg/extensions"
 	"github.com/anuvu/zot/pkg/log"
 	"github.com/anuvu/zot/pkg/storage"
+	"github.com/anuvu/zot/pkg/storage/s3"
 	"github.com/gorilla/handlers"
 	"github.com/gorilla/mux"
+
+	"github.com/docker/distribution/registry/storage/driver/factory"
 )
 
 const (
@@ -60,6 +63,7 @@ func DefaultHeaders() mux.MiddlewareFunc {
 	}
 }
 
+// nolint: gocyclo
 func (c *Controller) Run() error {
 	// validate configuration
 	if err := c.Config.Validate(c.Log); err != nil {
@@ -96,8 +100,25 @@ func (c *Controller) Run() error {
 			}
 		}
 
-		defaultStore := storage.NewImageStore(c.Config.Storage.RootDirectory,
-			c.Config.Storage.GC, c.Config.Storage.Dedupe, c.Log)
+		var defaultStore storage.ImageStore
+		if len(c.Config.Storage.StorageDriver) == 0 {
+			defaultStore = storage.NewImageStore(c.Config.Storage.RootDirectory,
+				c.Config.Storage.GC, c.Config.Storage.Dedupe, c.Log)
+		} else {
+			storeName := fmt.Sprintf("%v", c.Config.Storage.StorageDriver["name"])
+			if storeName != "s3" {
+				c.Log.Fatal().Err(errors.ErrBadConfig).Msg("zot supports only s3 storage driver")
+			}
+			// Init a Storager from connection string.
+			store, err := factory.Create(storeName, c.Config.Storage.StorageDriver)
+			if err != nil {
+				c.Log.Error().Err(err).Str("rootDir", c.Config.Storage.RootDirectory).Msg("Unable to create s3 service")
+				return err
+			}
+
+			defaultStore = s3.NewImageStore(c.Config.Storage.RootDirectory,
+				c.Config.Storage.GC, c.Config.Storage.Dedupe, c.Log, store)
+		}
 
 		c.StoreController.DefaultStore = defaultStore
 
@@ -130,8 +151,25 @@ func (c *Controller) Run() error {
 					}
 				}
 
-				subImageStore[route] = storage.NewImageStore(storageConfig.RootDirectory,
-					storageConfig.GC, storageConfig.Dedupe, c.Log)
+				if len(storageConfig.StorageDriver) == 0 {
+					subImageStore[route] = storage.NewImageStore(storageConfig.RootDirectory,
+						storageConfig.GC, storageConfig.Dedupe, c.Log)
+				} else {
+					storeName := fmt.Sprintf("%v", storageConfig.StorageDriver["name"])
+					if storeName != "s3" {
+						c.Log.Fatal().Err(errors.ErrBadConfig).Msg("zot supports only s3 storage driver")
+					}
+
+					// Init a Storager from connection string.
+					store, err := factory.Create(storeName, storageConfig.StorageDriver)
+					if err != nil {
+						c.Log.Error().Err(err).Str("rootDir", storageConfig.RootDirectory).Msg("Unable to create s3 service")
+						return err
+					}
+
+					subImageStore[route] = s3.NewImageStore(storageConfig.RootDirectory,
+						storageConfig.GC, storageConfig.Dedupe, c.Log, store)
+				}
 
 				// Enable extensions if extension config is provided
 				if c.Config != nil && c.Config.Extensions != nil {