From 254d34df32094af7893709ced8dc141d91d791ff Mon Sep 17 00:00:00 2001 From: Gleb Lesnikov Date: Fri, 7 Feb 2020 11:43:07 +0300 Subject: [PATCH] Add Azure Storage Backend (#89) * Initial azure backend implementationn * fix golangci-lint, docker-compose * [CI SKIP] excluded specific linter for magic numbers * Fixed error message in creating new container --- .drone.yml | 8 ++++++ CHANGELOG.md | 4 ++- DOCS.md | 13 +++++++-- README.md | 6 +++- cache/backend/azure.go | 50 +++++++++++++++++++++++++++++++++ cache/backend/azure_test.go | 53 +++++++++++++++++++++++++++++++++++ cache/backend/backend.go | 55 +++++++++++++++++++++++++++++++++++-- cache/backend/filesystem.go | 2 +- docker-compose.yml | 5 ++++ go.mod | 4 ++- go.sum | 33 ++++++++++++++++++++++ main.go | 34 +++++++++++++++++++++++ plugin/plugin.go | 4 +++ 13 files changed, 263 insertions(+), 8 deletions(-) create mode 100644 cache/backend/azure.go create mode 100644 cache/backend/azure_test.go diff --git a/.drone.yml b/.drone.yml index 7889da2b..9a895aa1 100644 --- a/.drone.yml +++ b/.drone.yml @@ -36,6 +36,7 @@ steps: CGO_ENABLED: 0 TEST_ENDPOINT: filestorage:9000 TEST_SFTP_HOST: sftp + TEST_AZURITE_URL: azurite:10000 volumes: - name: testcache path: /drone/src/testcache/cache @@ -275,6 +276,7 @@ steps: CGO_ENABLED: 0 TEST_ENDPOINT: filestorage:9000 TEST_SFTP_HOST: sftp + TEST_AZURITE_URL: azurite:10000 volumes: - name: testcache path: /drone/src/testcache/cache @@ -296,6 +298,12 @@ services: - 22 commands: - /entrypoint foo:pass:::upload +- name: azurite + image: mcr.microsoft.com/azure-storage/azurite + commands: + - azurite-blob --blobHost 0.0.0.0 + ports: + - 10000 volumes: - name: cache diff --git a/CHANGELOG.md b/CHANGELOG.md index 919d804e..140ee12b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Ureleased +## Unreleased + +- Azure Storage Backend. ### Added diff --git a/DOCS.md b/DOCS.md index 64eef4e5..54d1b8e6 100644 --- a/DOCS.md +++ b/DOCS.md @@ -2,7 +2,7 @@ date: 2019-03-19T00:00:00+00:00 title: Drone Cache author: meltwater -tags: [ cache, amazon, aws, s3, volume ] +tags: [ cache, amazon, aws, s3, azure, volume ] logo: drone_cache.svg repo: meltwater/drone-cache image: meltwater/drone-cache @@ -14,7 +14,7 @@ This plugin requires Volume configuration if you enable `filesystem` backend wit A Drone plugin for caching current workspace files between builds to reduce your build times. `drone-cache` is a small CLI program, written in Go without any external OS dependencies (such as tar, etc). -With `drone-cache`, you can provide your **own cache key templates**, specify **archive format** (tar, tar.gz, etc) and you can use **an S3 bucket or a mounted volume** as storage for your cached files, even better you can implement **your own storage backend** to cover your use case. +With `drone-cache`, you can provide your **own cache key templates**, specify **archive format** (tar, tar.gz, etc) and you can use **an S3 bucket, Azure Storage or a mounted volume** as storage for your cached files, even better you can implement **your own storage backend** to cover your use case. **How does it work** @@ -354,6 +354,15 @@ bucket region : AWS bucket region. (`us-east-1`, `eu-west-1`, ...) +account_name +: Azure Storage account name + +account_key +: Azure Storage account key + +container +: Azure Storage container + path-style : use path style for bucket paths. (true for `minio`, false for `aws`) diff --git a/README.md b/README.md index a39b92d5..74fc9beb 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ A Drone plugin for caching current workspace files between builds to reduce your build times. `drone-cache` is a small CLI program, written in Go without any external OS dependencies (such as tar, etc). -With `drone-cache`, you can provide your **own cache key templates**, specify **archive format** (tar, tar.gz, etc) and you can use **an S3 bucket or a mounted volume** as storage for your cached files, even better you can implement **your own storage backend** to cover your use case. +With `drone-cache`, you can provide your **own cache key templates**, specify **archive format** (tar, tar.gz, etc) and you can use **an S3 bucket, Azure Storage or a mounted volume** as storage for your cached files, even better you can implement **your own storage backend** to cover your use case. For detailed usage information and a list of available options please take a look at [usage](#usage) and [examples](#example-usage-of-drone-cache). If you want to learn more about custom cache keys, see [cache key templates](docs/cache_key_templates.md). @@ -154,6 +154,10 @@ GLOBAL OPTIONS: --path-style, --ps use path style for bucket paths. (true for minio, false for aws) [$PLUGIN_PATH_STYLE] --acl value upload files with acl (private, public-read, ...) (default: "private") [$PLUGIN_ACL] --encryption value, --enc value server-side encryption algorithm, defaults to none. (AES256, aws:kms) [$PLUGIN_ENCRYPTION] + --azure-account-name value Azure Blob Storage Account Name [$PLUGIN_ACCOUNT_NAME, $AZURE_ACCOUNT_NAME] + --azure-account-key value Azure Blob Storage Account Key [$PLUGIN_ACCOUNT_KEY, $AZURE_ACCOUNT_KEY] + --azure-container-name value Azure Blob Storage container name [$PLUGIN_CONTAINER, $AZURE_CONTAINER_NAME] + --azure-blob-storage-url value Azure Blob Storage URL (default: "blob.core.windows.net") [$AZURE_BLOB_STORAGE_URL] --sftp-cache-root value sftp root directory [$SFTP_CACHE_ROOT] --sftp-username value sftp username [$SFTP_USERNAME] --sftp-password value sftp password [$SFTP_PASSWORD] diff --git a/cache/backend/azure.go b/cache/backend/azure.go new file mode 100644 index 00000000..fd772597 --- /dev/null +++ b/cache/backend/azure.go @@ -0,0 +1,50 @@ +package backend + +import ( + "context" + "fmt" + "io" + + "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/meltwater/drone-cache/cache" +) + +type azureBackend struct { + containerURL azblob.ContainerURL + ctx context.Context +} + +func newAzure(ctx context.Context, containerURL azblob.ContainerURL) cache.Backend { + return &azureBackend{ + containerURL: containerURL, + ctx: ctx, + } +} + +func (c *azureBackend) Get(p string) (io.ReadCloser, error) { + blobURL := c.containerURL.NewBlockBlobURL(p) + + downloadResponse, err := blobURL.Download(c.ctx, 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false) + if err != nil { + return nil, fmt.Errorf("get the object %w", err) + } + + //nolint:mnd // NOTE: automatically retries are performed if the connection fails, not magic number + bodyStream := downloadResponse.Body(azblob.RetryReaderOptions{MaxRetryRequests: 4}) + + return bodyStream, nil +} + +// Put uploads the contents of the io.ReadSeeker +func (c *azureBackend) Put(p string, src io.ReadSeeker) error { + blobURL := c.containerURL.NewBlockBlobURL(p) + + fmt.Printf("uploading the file with blob name: %s\n", p) + + _, err := blobURL.Upload(c.ctx, src, azblob.BlobHTTPHeaders{}, azblob.Metadata{}, azblob.BlobAccessConditions{}) + if err != nil { + return fmt.Errorf("put the object %w", err) + } + + return nil +} diff --git a/cache/backend/azure_test.go b/cache/backend/azure_test.go new file mode 100644 index 00000000..70b2835f --- /dev/null +++ b/cache/backend/azure_test.go @@ -0,0 +1,53 @@ +package backend + +import ( + "bytes" + "io/ioutil" + "math/rand" + "testing" + + "github.com/go-kit/kit/log" +) + +const defaultBlobStorageURL = "127.0.0.1:10000" + +var blobURL = getEnv("TEST_AZURITE_URL", defaultBlobStorageURL) + +func TestAzureTruth(t *testing.T) { + + b, err := InitializeAzureBackend(log.NewNopLogger(), + AzureConfig{ + AccountName: "devstoreaccount1", + AccountKey: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + ContainerName: "testcontainer", + BlobStorageURL: blobURL, + Azurite: true, + }, true) + if err != nil { + t.Fatal(err) + } + + token := make([]byte, 32) + rand.Read(token) + testData := bytes.NewReader(token) + + // PUT TEST + err = b.Put("test_key", testData) + if err != nil { + t.Fatal(err) + } + + // GET TEST + readCloser, err := b.Get("test_key") + if err != nil { + t.Fatal(err) + } + + // Check the validity of returned bytes + readData, _ := ioutil.ReadAll(readCloser) + + if !bytes.Equal(readData, token) { + t.Fatal(string(readData), "!=", token) + } + +} diff --git a/cache/backend/backend.go b/cache/backend/backend.go index 88aae1bf..8b022944 100644 --- a/cache/backend/backend.go +++ b/cache/backend/backend.go @@ -1,19 +1,21 @@ package backend import ( + "context" "errors" "fmt" "io/ioutil" + "net/url" "os" "path" "strings" - "github.com/meltwater/drone-cache/cache" - + "github.com/Azure/azure-storage-blob-go/azblob" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" + "github.com/meltwater/drone-cache/cache" "github.com/pkg/sftp" "golang.org/x/crypto/ssh" ) @@ -48,6 +50,15 @@ type S3Config struct { PathStyle bool // Use path style instead of domain style. Should be true for minio and false for AWS } +// AzureConfig is a structure to store Azure backend configuration +type AzureConfig struct { + AccountName string + AccountKey string + ContainerName string + BlobStorageURL string + Azurite bool +} + // FileSystemConfig is a structure to store filesystem backend configuration type FileSystemConfig struct { CacheRoot string @@ -77,6 +88,46 @@ func InitializeS3Backend(l log.Logger, c S3Config, debug bool) (cache.Backend, e return newS3(c.Bucket, c.ACL, c.Encryption, awsConf), nil } +// InitializeAzureBackend creates an AzureBlob backend +func InitializeAzureBackend(l log.Logger, c AzureConfig, debug bool) (cache.Backend, error) { + // From the Azure portal, get your storage account name and key and set environment variables. + accountName, accountKey := c.AccountName, c.AccountKey + if len(accountName) == 0 || len(accountKey) == 0 { + return nil, fmt.Errorf("either the AZURE_ACCOUNT_NAME or AZURE_ACCOUNT_KEY environment variable is not set") + } + + // Create a default request pipeline using your storage account name and account key. + credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) + if err != nil { + level.Error(l).Log("msg", "invalid credentials with error: "+err.Error()) + } + + var azureBlobURL *url.URL + + // Azurite has different URL pattern than production Azure Blob Storage + if c.Azurite { + azureBlobURL, err = url.Parse(fmt.Sprintf("http://%s/%s/%s", c.BlobStorageURL, c.AccountName, c.ContainerName)) + } else { + azureBlobURL, err = url.Parse(fmt.Sprintf("https://%s.%s/%s", c.AccountName, c.BlobStorageURL, c.ContainerName)) + } + + if err != nil { + level.Error(l).Log("msg", "can't create url with : "+err.Error()) + } + + pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{}) + containerURL := azblob.NewContainerURL(*azureBlobURL, pipeline) + ctx := context.Background() + + // Always creating new container, it will throw error if it already exists + _, err = containerURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone) + if err != nil { + level.Debug(l).Log("msg", "container already exists:"+err.Error()) + } + + return newAzure(ctx, containerURL), nil +} + // InitializeFileSystemBackend creates a filesystem backend func InitializeFileSystemBackend(l log.Logger, c FileSystemConfig, debug bool) (cache.Backend, error) { if strings.TrimRight(path.Clean(c.CacheRoot), "/") == "" { diff --git a/cache/backend/filesystem.go b/cache/backend/filesystem.go index 1e833ac4..df4238fb 100644 --- a/cache/backend/filesystem.go +++ b/cache/backend/filesystem.go @@ -37,7 +37,7 @@ func (c *filesystem) Put(p string, src io.ReadSeeker) error { } dir := filepath.Dir(absPath) - if err := os.MkdirAll(dir, os.FileMode(0755)); err != nil { + if err := os.MkdirAll(dir, os.FileMode(0755)); err != nil { //nolint:mnd 755 is not a magic number return fmt.Errorf("create directory <%s> %w", dir, err) } diff --git a/docker-compose.yml b/docker-compose.yml index 19151349..f5095133 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,11 @@ services: ports: - "22:22" command: foo:pass:::upload + azurite: + image: mcr.microsoft.com/azure-storage/azurite + ports: + - "10000:10000" + command: azurite-blob --blobHost 0.0.0.0 configure-buckets: image: minio/mc:RELEASE.2018-09-26T00-42-43Z entrypoint: sh diff --git a/go.mod b/go.mod index e4b8d8ec..a7976760 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/meltwater/drone-cache require ( + github.com/Azure/azure-storage-blob-go v0.8.0 + github.com/Azure/go-autorest/autorest/adal v0.8.1 // indirect github.com/aws/aws-sdk-go v1.16.35 github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-ini/ini v1.41.0 // indirect @@ -15,7 +17,7 @@ require ( github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 // indirect github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c // indirect github.com/urfave/cli v1.20.0 - golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 + golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 golang.org/x/sys v0.0.0-20191008105621-543471e840be // indirect gopkg.in/ini.v1 v1.41.0 // indirect ) diff --git a/go.sum b/go.sum index 2a7dabe1..29f0b919 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,30 @@ +github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo= +github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= +github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o= +github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0= +github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.1 h1:pZdL8o72rK+avFWl+p9nE8RWi1JInZrWJYlnpfXJwHk= +github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/aws/aws-sdk-go v1.16.35 h1:qz1h7uxswkVaE6kJPoPWwt3F76HlCLrg/UyDJq3cavc= github.com/aws/aws-sdk-go v1.16.35/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/go-ini/ini v1.41.0 h1:526aoxDtxRHFQKMZfcX2OG9oOI8TJ5yPLM0Mkno/uTY= github.com/go-ini/ini v1.41.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= @@ -21,6 +43,13 @@ github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149 h1:HfxbT6/JcvIljmERptWhwa8XzP7H3T+Z2N26gTsaDaA= +github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -43,6 +72,8 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -54,6 +85,8 @@ golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.41.0 h1:Ka3ViY6gNYSKiVy71zXBEqKplnV35ImDLVG+8uoIklE= gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= diff --git a/main.go b/main.go index 192b6e94..05dd3e15 100644 --- a/main.go +++ b/main.go @@ -323,6 +323,32 @@ func main() { EnvVar: "PLUGIN_ENCRYPTION", }, + // Azure specific Config flags + + cli.StringFlag{ + Name: "azure-account-name", + Usage: "Azure Blob Storage Account Name", + EnvVar: "PLUGIN_ACCOUNT_NAME,AZURE_ACCOUNT_NAME", + }, + cli.StringFlag{ + Name: "azure-account-key", + Usage: "Azure Blob Storage Account Key", + EnvVar: "PLUGIN_ACCOUNT_KEY,AZURE_ACCOUNT_KEY", + }, + cli.StringFlag{ + Name: "azure-container-name", + Usage: "Azure Blob Storage container name", + EnvVar: "PLUGIN_CONTAINER,AZURE_CONTAINER_NAME", + }, + cli.StringFlag{ + Name: "azure-blob-storage-url", + Usage: "Azure Blob Storage URL", + Value: "blob.core.windows.net", + EnvVar: "AZURE_BLOB_STORAGE_URL", + }, + + // SFTP specific Config flags + cli.StringFlag{ Name: "sftp-cache-root", Usage: "sftp root directory", @@ -434,6 +460,13 @@ func run(c *cli.Context) error { Region: c.String("region"), Secret: c.String("secret-key"), }, + Azure: backend.AzureConfig{ + AccountName: c.String("azure-account-name"), + AccountKey: c.String("azure-account-key"), + ContainerName: c.String("azure-container-name"), + BlobStorageURL: c.String("azure-blob-storage-url"), + Azurite: false, + }, SFTP: backend.SFTPConfig{ CacheRoot: c.String("sftp-cache-root"), Username: c.String("sftp-username"), @@ -445,6 +478,7 @@ func run(c *cli.Context) error { Method: backend.SSHAuthMethod(c.String("sftp-auth-method")), }, }, + SkipSymlinks: c.Bool("skip-symlinks"), }, } diff --git a/plugin/plugin.go b/plugin/plugin.go index 4d708859..0e26ff83 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -36,6 +36,7 @@ type ( S3 backend.S3Config FileSystem backend.FileSystemConfig SFTP backend.SFTPConfig + Azure backend.AzureConfig } // Plugin stores metadata about current plugin. @@ -108,6 +109,9 @@ func (p *Plugin) Exec() error { // initializeBackend initializes backend using given configuration func initializeBackend(logger log.Logger, c Config) (cache.Backend, error) { switch c.Backend { + case "azure": + level.Warn(logger).Log("msg", "using azure blob as backend") + return backend.InitializeAzureBackend(logger, c.Azure, c.Debug) case "s3": level.Warn(logger).Log("msg", "using aws s3 as backend") return backend.InitializeS3Backend(logger, c.S3, c.Debug)