[ATEAM-13094]: Implement Alibaba OSS backend

CHANGELOG.md

@@ -9,8 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- [#223](https://github.com/meltwater/drone-cache/pull/223) Added implementation for AlibabaOSS for backend storage
+
 ### Changed
 
+- Updated dependency `cloud.google.com/go/storage v1.24.0` -> `v1.26.0`
+- Updated dependency `github.com/urfave/cli/v2 v2.11.1` -> `v2.14.1`
+- Updated dependency `google.golang.org/api v0.88.0` -> `v0.94.0`
+- Updated dependency `google.golang.org/protobuf v1.28.0 ` -> `v1.28.1`
+
 ### Removed
 
 ## [1.4.0] - 2022-09-01

README.md

@@ -124,6 +124,8 @@ COMMANDS:
 GLOBAL OPTIONS:
    --access-key value                    AWS access key [$PLUGIN_ACCESS_KEY, $AWS_ACCESS_KEY_ID, $CACHE_AWS_ACCESS_KEY_ID]
    --acl value                           upload files with acl (private, public-read, ...) (default: "private") [$PLUGIN_ACL, $AWS_ACL]
+   --alibaba.access-key value            AlibabaOSS access key [$PLUGIN_ALIBABA_ACCESS_KEY, $ALIBABA_ACCESS_KEY_ID, $CACHE_ALIBABA_ACCESS_KEY_ID]
+   --alibaba.secret-key value            AlibabaOSS access secret [$PLUGIN_ALIBABA_ACCESS_SECRET, $ALIBABA_ACCESS_SECRET, $CACHE_ALIBABA_ACCESS_SECRET]
    --archive-format value                archive format to use to store the cache directories (tar, gzip, zstd) (default: "tar") [$PLUGIN_ARCHIVE_FORMAT]
    --azure.account-key value             Azure Blob Storage Account Key [$PLUGIN_ACCOUNT_KEY, $AZURE_ACCOUNT_KEY]
    --azure.account-name value            Azure Blob Storage Account Name [$PLUGIN_ACCOUNT_NAME, $AZURE_ACCOUNT_NAME]

go.mod

@@ -3,21 +3,25 @@ module github.com/meltwater/drone-cache
 go 1.18
 
 require (
-	cloud.google.com/go/storage v1.24.0
+	cloud.google.com/go/storage v1.26.0
 	github.com/Azure/azure-storage-blob-go v0.15.0
-	github.com/aws/aws-sdk-go v1.44.60
+	github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible
+	github.com/aws/aws-sdk-go v1.44.92
 	github.com/bmatcuk/doublestar/v4 v4.2.0
 	github.com/dustin/go-humanize v1.0.0
 	github.com/go-kit/log v0.2.1
 	github.com/google/go-cmp v0.5.8
 	github.com/klauspost/compress v1.15.9
+	github.com/pkg/errors v0.9.1
 	github.com/pkg/sftp v1.13.5
-	github.com/urfave/cli/v2 v2.11.1
+	github.com/urfave/cli/v2 v2.14.1
 	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
-	golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c
-	google.golang.org/api v0.88.0
+	golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094
+	google.golang.org/api v0.94.0
 )
 
+require github.com/mattn/go-ieproxy v0.0.7 // indirect
+
 require (
 	cloud.google.com/go v0.103.0 // indirect
 	cloud.google.com/go/compute v1.7.0 // indirect
@@ -33,18 +37,18 @@ require (
 	github.com/googleapis/gax-go/v2 v2.4.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kr/fs v0.1.0 // indirect
-	github.com/mattn/go-ieproxy v0.0.7 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/stretchr/testify v1.7.1 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	go.opencensus.io v0.23.0 // indirect
-	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
-	golang.org/x/sys v0.0.0-20220721230656-c6bc011c0c49 // indirect
+	golang.org/x/net v0.0.0-20220906165146-f3363e06e74c // indirect
+	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
 	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220720214146-176da50484ac // indirect
+	google.golang.org/genproto v0.0.0-20220810155839-1856144b1d9c // indirect
 	google.golang.org/grpc v1.48.0 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

internal/plugin/config.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/bmatcuk/doublestar/v4"
+	"github.com/meltwater/drone-cache/storage/backend/alioss"
 	"github.com/meltwater/drone-cache/storage/backend/azure"
 	"github.com/meltwater/drone-cache/storage/backend/filesystem"
 	"github.com/meltwater/drone-cache/storage/backend/gcs"
@@ -41,6 +42,7 @@ type Config struct {
 	SFTP       sftp.Config
 	Azure      azure.Config
 	GCS        gcs.Config
+	Alioss     alioss.Config
 }
 
 // HandleMount runs prior to Rebuild and Restoring of caches to handle unique

internal/plugin/plugin.go

@@ -106,6 +106,7 @@ func (p *Plugin) Exec() error { // nolint: funlen,cyclop
 		GCS:        cfg.GCS,
 		S3:         cfg.S3,
 		SFTP:       cfg.SFTP,
+		Alioss:     cfg.Alioss,
 	})
 	if err != nil {
 		return fmt.Errorf("initialize backend <%s>, %w", cfg.Backend, err)

main.go

@@ -14,6 +14,7 @@ import (
 	"github.com/meltwater/drone-cache/internal/plugin"
 	"github.com/meltwater/drone-cache/storage"
 	"github.com/meltwater/drone-cache/storage/backend"
+	"github.com/meltwater/drone-cache/storage/backend/alioss"
 	"github.com/meltwater/drone-cache/storage/backend/azure"
 	"github.com/meltwater/drone-cache/storage/backend/filesystem"
 	"github.com/meltwater/drone-cache/storage/backend/gcs"
@@ -474,6 +475,19 @@ func main() {
 			Usage:   "sftp port",
 			EnvVars: []string{"SFTP_PORT"},
 		},
+
+		// Alibaba OSS (storage) specific Config flags
+
+		&cli.StringFlag{
+			Name:    "alibaba.access-key",
+			Usage:   "AlibabaOSS access key",
+			EnvVars: []string{"PLUGIN_ALIBABA_ACCESS_KEY", "ALIBABA_ACCESS_KEY_ID", "CACHE_ALIBABA_ACCESS_KEY_ID"},
+		},
+		&cli.StringFlag{
+			Name:    "alibaba.secret-key",
+			Usage:   "AlibabaOSS access secret",
+			EnvVars: []string{"PLUGIN_ALIBABA_ACCESS_SECRET", "ALIBABA_ACCESS_SECRET", "CACHE_ALIBABA_ACCESS_SECRET"},
+		},
 	}
 
 	if err := app.Run(os.Args); err != nil {
@@ -586,6 +600,12 @@ func run(c *cli.Context) error {
 			Encryption: c.String("gcs.encryption-key"),
 			Timeout:    c.Duration("backend.operation-timeout"),
 		},
+		Alioss: alioss.Config{
+			Bucket:         c.String("bucket"),
+			Endpoint:       c.String("endpoint"),
+			AccesKeyID:     c.String("alibaba.access-key"),
+			AccesKeySecret: c.String("alibaba.secret-key"),
+		},
 
 		SkipSymlinks: c.Bool("skip-symlinks"),
 	}

storage/backend/alioss/alioss.go

@@ -0,0 +1,114 @@
+package alioss
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
+	"github.com/pkg/errors"
+)
+
+// Backend implements storage.Backend for Alibaba OSS storage.
+type Backend struct {
+	logger log.Logger
+
+	bucket     string
+	acl        string
+	encryption string
+	client     *oss.Client
+}
+
+func newAlibabaOss(bucket string, conf *oss.Config, opts ...oss.ClientOption) (*Backend, error) {
+	client, err := oss.New(conf.Endpoint, conf.AccessKeyID, conf.AccessKeySecret, opts...)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create alibabaOSS client")
+	}
+
+	return &Backend{
+		bucket: bucket,
+		client: client,
+	}, nil
+}
+
+func New(l log.Logger, c Config, debug bool) (*Backend, error) {
+	ossConf := &oss.Config{}
+	if c.Endpoint != "" {
+		ossConf.Endpoint = c.Endpoint
+	}
+
+	if c.AccesKeyID != "" {
+		ossConf.AccessKeyID = c.AccesKeyID
+	}
+
+	if c.AccesKeySecret != "" {
+		ossConf.AccessKeySecret = c.AccesKeySecret
+	}
+
+	if debug {
+		level.Debug(l).Log("msg", "oss storage backend", "config", fmt.Sprintf("%+v", c))
+	}
+
+	return newAlibabaOss(c.Bucket, ossConf)
+}
+
+func (c Backend) Get(ctx context.Context, p string, w io.Writer) error {
+	bucket, err := c.client.Bucket(c.bucket)
+	if err != nil {
+		return errors.Wrap(err, "couldn't get the object")
+	}
+
+	reader, err := bucket.GetObject(p)
+	if err != nil {
+		return errors.Wrap(err, "couldn't get the object")
+	}
+
+	if reader != nil {
+		level.Info(c.logger).Log("Successfully read fileobject")
+	}
+
+	return nil
+}
+
+func (c Backend) Put(ctx context.Context, p string, src io.Reader) error {
+	bucket, err := c.client.Bucket(c.bucket)
+	if err != nil {
+		return errors.Wrap(err, "couldn't put the object")
+	}
+
+	options := []oss.Option{}
+
+	if c.encryption != "" {
+		option := oss.ServerSideEncryption(c.encryption)
+		options = append(options, option)
+	}
+
+	if c.acl != "" {
+		option := oss.ObjectACL(oss.ACLType(c.acl))
+		options = append(options, option)
+	}
+
+	if err := bucket.PutObject(p, src, options...); err != nil {
+		return errors.Wrap(err, "couldn't put the object")
+	}
+
+	return nil
+}
+
+func (c Backend) Exists(ctx context.Context, p string) (bool, error) {
+	bucket, err := c.client.Bucket(c.bucket)
+	if err != nil {
+		return false, errors.Wrap(err, "couldn't get the bucket object")
+	}
+
+	options := []oss.Option{}
+
+	result, err := bucket.IsObjectExist(p, options...)
+	if err != nil {
+		return false, errors.Wrap(err, "couldn't get the object")
+	}
+
+	return result, nil
+}

storage/backend/alioss/alioss_test.go

@@ -0,0 +1,7 @@
+package alioss
+
+import "testing"
+
+func TestAlibabaOss(t *testing.T) {
+	t.Skip("skipping backend package tests")
+}

storage/backend/alioss/config.go

@@ -0,0 +1,9 @@
+package alioss
+
+// Config is a structure to store AlibabaOss backend configuration.
+type Config struct {
+	Bucket         string
+	Endpoint       string
+	AccesKeyID     string
+	AccesKeySecret string
+}

storage/backend/backend.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/meltwater/drone-cache/storage/backend/alioss"
 	"github.com/meltwater/drone-cache/storage/backend/azure"
 	"github.com/meltwater/drone-cache/storage/backend/filesystem"
 	"github.com/meltwater/drone-cache/storage/backend/gcs"
@@ -27,6 +28,8 @@ const (
 	S3 = "s3"
 	// SFTP type of the corresponding backend represented as string constant.
 	SFTP = "sftp"
+	// OSS type of the corresponding backend represented as string constant.
+	AliOSS = "alioss"
 )
 
 // MOTICE: FileEntry needs a better place.
@@ -79,6 +82,9 @@ func FromConfig(l log.Logger, backedType string, cfg Config) (Backend, error) {
 	case SFTP:
 		level.Warn(l).Log("msg", "using sftp as backend")
 		b, err = sftp.New(log.With(l, "backend", SFTP), cfg.SFTP)
+	case AliOSS:
+		level.Warn(l).Log("msg", "using Alibaba OSS storage as backend")
+		b, err = alioss.New(log.With(l, "backend", AliOSS), cfg.Alioss, cfg.Debug)
 	default:
 		return nil, errors.New("unknown backend")
 	}

storage/backend/config.go

@@ -1,6 +1,7 @@
 package backend
 
 import (
+	"github.com/meltwater/drone-cache/storage/backend/alioss"
 	"github.com/meltwater/drone-cache/storage/backend/azure"
 	"github.com/meltwater/drone-cache/storage/backend/filesystem"
 	"github.com/meltwater/drone-cache/storage/backend/gcs"
@@ -17,4 +18,5 @@ type Config struct {
 	SFTP       sftp.Config
 	Azure      azure.Config
 	GCS        gcs.Config
+	Alioss     alioss.Config
 }