✨ Implement S3 multipart upload

Signed-off-by: Rintaro Okamura <rintaro.okamura@gmail.com>

internal/config/blob.go

@@ -45,8 +45,8 @@ type Blob struct {
 	// StorageType represents blob storaget type
 	StorageType string `json:"storage_type" yaml:"storage_type"`
 
-	// BucketURL represents bucket URL
-	BucketURL string `json:"bucket_url" yaml:"bucket_url"`
+	// Bucket represents bucket name
+	Bucket string `json:"bucket" yaml:"bucket"`
 
 	// S3 represents S3 config
 	S3 *S3Config `json:"s3" yaml:"s3"`
@@ -63,7 +63,7 @@ type S3Config struct {
 
 func (b *Blob) Bind() *Blob {
 	b.StorageType = GetActualValue(b.StorageType)
-	b.BucketURL = GetActualValue(b.BucketURL)
+	b.Bucket = GetActualValue(b.Bucket)
 
 	if b.S3 != nil {
 		b.S3 = b.S3.Bind()

internal/config/sidecar.go

@@ -18,8 +18,6 @@
 package config
 
 type AgentSidecar struct {
-	// Name string `yaml:"name" json:"name"`
-
 	// WatchPaths represents watch path list for backup
 	WatchPaths []string `yaml:"watch_paths" json:"watch_paths"`
 
@@ -28,11 +26,25 @@ type AgentSidecar struct {
 
 	// AutoBackupDuration represent checking loop duration for auto backup execution
 	AutoBackupDuration string `yaml:"auto_backup_duration" json:"auto_backup_duration"`
+
+	// Filename represent backup filename
+	Filename string `yaml:"filename" json:"filename"`
+
+	// BlobStorage represent blob storage configurations
+	BlobStorage *Blob `yaml:"blob_storage" json:"blob_storage"`
 }
 
 func (s *AgentSidecar) Bind() *AgentSidecar {
 	s.WatchPaths = GetActualValues(s.WatchPaths)
 	s.AutoBackupDuration = GetActualValue(s.AutoBackupDuration)
 	s.AutoBackupDurationLimit = GetActualValue(s.AutoBackupDurationLimit)
+	s.Filename = GetActualValue(s.Filename)
+
+	if s.BlobStorage != nil {
+		s.BlobStorage = s.BlobStorage.Bind()
+	} else {
+		s.BlobStorage = new(Blob)
+	}
+
 	return s
 }

internal/db/storage/blob/blob.go

@@ -19,60 +19,11 @@ package blob
 import (
 	"context"
 	"io"
-	"net/url"
-	"reflect"
-
-	"github.com/vdaas/vald/internal/errors"
-
-	"gocloud.dev/blob"
 )
 
-type BucketURLOpener = blob.BucketURLOpener
-
-type bucket struct {
-	opener BucketURLOpener
-	url    string
-	bucket *blob.Bucket
-}
-
 type Bucket interface {
 	Open(ctx context.Context) error
 	Close() error
 	Reader(ctx context.Context, key string) (io.ReadCloser, error)
 	Writer(ctx context.Context, key string) (io.WriteCloser, error)
 }
-
-func NewBucket(opts ...Option) (Bucket, error) {
-	b := new(bucket)
-	for _, opt := range append(defaultOpts, opts...) {
-		if err := opt(b); err != nil {
-			return nil, errors.ErrOptionFailed(err, reflect.ValueOf(opt))
-		}
-	}
-
-	return b, nil
-}
-
-func (b *bucket) Open(ctx context.Context) (err error) {
-	url, err := url.Parse(b.url)
-	if err != nil {
-		return err
-	}
-	b.bucket, err = b.opener.OpenBucketURL(ctx, url)
-	return err
-}
-
-func (b *bucket) Close() error {
-	if b.bucket != nil {
-		return b.bucket.Close()
-	}
-	return nil
-}
-
-func (b *bucket) Reader(ctx context.Context, key string) (io.ReadCloser, error) {
-	return b.bucket.NewReader(ctx, key, nil)
-}
-
-func (b *bucket) Writer(ctx context.Context, key string) (io.WriteCloser, error) {
-	return b.bucket.NewWriter(ctx, key, nil)
-}