diff --git a/.drone.yml b/.drone.yml index 9a415649..bc12b5cf 100644 --- a/.drone.yml +++ b/.drone.yml @@ -24,14 +24,14 @@ steps: - name: build pull: true - image: golang:1.11-alpine + image: golang:1.13-alpine commands: - apk add --update make git - make drone-cache - name: test pull: true - image: golang:1.11-alpine + image: golang:1.13-alpine commands: - go test -v -mod=vendor -cover ./... environment: @@ -42,17 +42,17 @@ steps: - name: testcache path: /drone/src/testcache/cache -- name: analyze +- name: lint pull: true - image: golang:1.11-alpine + image: golang:1.13-alpine commands: - - "wget -O - -q https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.14.0" + - "wget -O - -q https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.21.0" - ./bin/golangci-lint run -v --enable-all -D gochecknoglobals environment: CGO_ENABLED: 0 - name: release-snapshot-dev - image: goreleaser/goreleaser:v0.109 + image: goreleaser/goreleaser:v0.120 commands: - apk add --update make upx - goreleaser release --rm-dist --snapshot @@ -149,7 +149,7 @@ steps: path: /tmp/cache - name: release-snapshot - image: goreleaser/goreleaser:v0.109 + image: goreleaser/goreleaser:v0.120 commands: - apk add --update make upx - goreleaser release --rm-dist --snapshot @@ -264,14 +264,14 @@ steps: - name: build-after pull: true - image: golang:1.11-alpine + image: golang:1.13-alpine commands: - apk add --update make git - make drone-cache - name: test-after pull: true - image: golang:1.11-alpine + image: golang:1.13-alpine commands: - go test -v -mod=vendor -cover ./... environment: @@ -328,10 +328,10 @@ steps: - git fetch --tags - name: release - image: goreleaser/goreleaser:v0.109 + image: goreleaser/goreleaser:v0.120 commands: - apk add --update make upx - - goreleaser release + - make release environment: GITHUB_TOKEN: from_secret: github_token diff --git a/.errcheck_excludes b/.errcheck_excludes new file mode 100644 index 00000000..97b97714 --- /dev/null +++ b/.errcheck_excludes @@ -0,0 +1 @@ +(github.com/go-kit/kit/log.Logger).Log diff --git a/.gitignore b/.gitignore index 14c0d1a2..77646b9a 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ target bin testcache backup +tmp diff --git a/.golangci.yml b/.golangci.yml index 075da97f..f36f3c8b 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -5,3 +5,14 @@ run: deadline: 1m tests: false modules-download-mode: vendor + +linters: + enable-all: true + disable: + - gochecknoglobals + +linters-settings: + errcheck: + exclude: .errcheck_excludes + lll: + line-length: 120 diff --git a/.goreleaser.yml b/.goreleaser.yml index ef643200..1bf9cd64 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,7 +1,7 @@ before: hooks: - make clean - - make fetch-dependencies + - make vendor dist: target/dist builds: - env: diff --git a/CHANGELOG.md b/CHANGELOG.md index 10be9524..919d804e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +[#84](https://github.com/meltwater/drone-cache/pull/84) Adds compression level option. [#77](https://github.com/meltwater/drone-cache/pull/77) Adds a new CLI hidden flag to be used for tests. [#68](https://github.com/meltwater/drone-cache/pull/68) Introduces new storage backend, sFTP. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9e72e9ef..afa3e02f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,13 +15,14 @@ the requirements below. ## Pull Request Process 0. Check out [Pull Request Checklist](#pull-request-checklist), ensure you have fulfilled each step. -1. Ensure any install or build dependencies are removed before the end of the layer when doing a +1. Check out [Uber Style Guide](https://github.com/uber-go/guide/blob/master/style.md), project tries to follow it, ensure you have fulfilled it as much as possible. +2. Ensure any install or build dependencies are removed before the end of the layer when doing a build. -2. Please ensure the [README](README.md) and [DOCS](./DOCS.md) are up-to-date with details of changes to the command-line interface, +3. Please ensure the [README](README.md) and [DOCS](./DOCS.md) are up-to-date with details of changes to the command-line interface, this includes new environment variables, exposed ports, useful file locations and container parameters. -3. **PLEASE ENSURE YOU DO NOT INTRODUCE BREAKING CHANGES.** -4. **PLEASE ENSURE BUG FIXES AND NEW FEATURES INCLUDE TESTS.** -5. You may merge the Pull Request in once you have the sign-off of one other maintainer/code owner, +4. **PLEASE ENSURE YOU DO NOT INTRODUCE BREAKING CHANGES.** +5. **PLEASE ENSURE BUG FIXES AND NEW FEATURES INCLUDE TESTS.** +6. You may merge the Pull Request in once you have the sign-off of one other maintainer/code owner, or if you do not have permission to do that, you may request the second reviewer to merge it for you. ## Pull Request Checklist @@ -43,12 +44,13 @@ the requirements below. *Only concerns maintainers/code owners* 0. **PLEASE DO NOT INTRODUCE BREAKING CHANGES** -1. Increase the version numbers in any examples files and the README.md to the new version that this +1. Execute `make README.md`. This will update [usage](README.md#usage) section of [README.md](README.md) with latest CLI options +2. Increase the version numbers in any examples files and the README.md to the new version that this release would represent. The versioning scheme we use is [SemVer](http://semver.org/) for versioning. For the versions available, see the [tags on this repository](https://github.com/meltwater/drone-cache/tags). -2. Ensure [CHANGELOG](CHANGELOG.md) is up-to-date with new version changes. -3. Update version references. -4. Create a tag on master. Any changes on master will trigger a release with given tag and `latest tag. +3. Ensure [CHANGELOG](CHANGELOG.md) is up-to-date with new version changes. +4. Update version references. +5. Create a tag on master. Any changes on master will trigger a release with given tag and `latest tag. ```console $ git tag -am 'vX.X.X' diff --git a/DOCS.md b/DOCS.md index eb127892..64eef4e5 100644 --- a/DOCS.md +++ b/DOCS.md @@ -115,7 +115,7 @@ steps: - 'vendor' - name: build - image: golang:1.11-alpine + image: golang:1.13-alpine pull: true commands: - apk add --update make git @@ -160,7 +160,7 @@ steps: path: /tmp/cache - name: build - image: golang:1.11-alpine + image: golang:1.13-alpine pull: true commands: - apk add --update make git @@ -212,7 +212,7 @@ steps: - 'vendor' - name: build - image: golang:1.11-alpine + image: golang:1.13-alpine pull: true commands: - apk add --update make git @@ -260,7 +260,7 @@ steps: - 'vendor' - name: build - image: golang:1.11-alpine + image: golang:1.13-alpine pull: true commands: - apk add --update make git @@ -299,7 +299,7 @@ steps: debug: true - name: build - image: golang:1.11-alpine + image: golang:1.13-alpine pull: true commands: - apk add --update make git diff --git a/Dockerfile b/Dockerfile index 69ba54a6..9664ed70 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.11-alpine AS builder +FROM golang:1.13-alpine AS builder RUN apk add --update --no-cache ca-certificates tzdata && update-ca-certificates RUN echo "[WARNING] Make sure you have run 'goreleaser release', before 'docker build'!" diff --git a/Makefile b/Makefile index 970efdb4..09792573 100644 --- a/Makefile +++ b/Makefile @@ -2,45 +2,73 @@ VERSION := $(strip $(shell [ -d .git ] && git describe --always --tags --dirty)) BUILD_DATE := $(shell date -u +"%Y-%m-%dT%H:%M:%S%Z") VCS_REF := $(strip $(shell [ -d .git ] && git rev-parse --short HEAD)) +GO_PACKAGES=$(shell go list ./... | grep -v -E '/vendor/|/test') +GO_FILES:=$(shell find . -name \*.go -print) +GOPATH:=$(firstword $(subst :, ,$(shell go env GOPATH))) + +GOLANGCI_LINT_VERSION=v1.21.0 +GOLANGCI_LINT_BIN=$(GOPATH)/bin/golangci-lint +EMBEDMD_BIN=$(GOPATH)/bin/embedmd +GOTEST_BIN=$(GOPATH)/bin/gotest + +.PHONY: default all default: drone-cache all: drone-cache -drone-cache: fetch-dependencies main.go $(wildcard *.go) $(wildcard */*.go) - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -mod=vendor -a -ldflags '-s -w' -o $@ . +drone-cache: vendor main.go $(wildcard *.go) $(wildcard */*.go) + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -mod=vendor -a -ldflags '-s -w -X main.version=$(VERSION)' -o $@ . -build: fetch-dependencies main.go $(wildcard *.go) $(wildcard */*.go) - go build -mod=vendor -a -ldflags '-s -w' -o drone-cache . +.PHONY: build +build: vendor main.go $(wildcard *.go) $(wildcard */*.go) + go build -mod=vendor -a -ldflags '-s -w -X main.version=$(VERSION)' -o drone-cache . -release: +.PHONY: release +release: build goreleaser release --rm-dist +.PHONY: snapshot snapshot: goreleaser release --skip-publish --rm-dist --snapshot +.PHONY: clean clean: rm -f drone-cache rm -rf target -.PHONY: default all clean release snapshot +tmp/help.txt: clean build + mkdir -p tmp + ./drone-cache --help &> tmp/help.txt -fetch-dependencies: - @go mod vendor -v +README.md: tmp/help.txt + embedmd -w README.md -.PHONY: fetch-dependencies +tmp/docs.txt: clean build + mkdir -p tmp + # ./drone-cache --help &> tmp/help.txt + @echo "IMPLEMENT ME" -build-compressed: drone-cache - @upx drone-cache +DOCS.md: tmp/docs.txt + embedmd -w DOCS.md + +.PHONY: vendor +vendor: + @go mod tidy + @go mod vendor -v -.PHONY: build-compressed +.PHONY: compress +compress: drone-cache + @upx drone-cache -docker-build: release Dockerfile +.PHONY: container +container: release Dockerfile @docker build --build-arg BUILD_DATE="$(BUILD_DATE)" \ --build-arg VERSION="$(VERSION)" \ --build-arg VCS_REF="$(VCS_REF)" \ --build-arg DOCKERFILE_PATH="/Dockerfile" \ -t meltwater/drone-cache:latest . -docker-build-dev: snapshot Dockerfile +.PHONY: container-dev +container-dev: snapshot Dockerfile @docker build --build-arg BUILD_DATE="$(BUILD_DATE)" \ --build-arg VERSION="$(VERSION)" \ --build-arg VCS_REF="$(VCS_REF)" \ @@ -48,18 +76,40 @@ docker-build-dev: snapshot Dockerfile --no-cache \ -t meltwater/drone-cache:dev . -docker-push: docker-build +.PHONY: container-push +container-push: container docker push meltwater/drone-cache:latest -docker-push-dev: docker-build-dev +.PHONY: container-push-dev +container-push-dev: container-dev docker push meltwater/drone-cache:dev -.PHONY: docker-build docker-push +.PHONY: test +test: $(GOTEST_BIN) + docker-compose up -d + mkdir -p ./testcache/cache + gotest -race -short -cover ./... + +.PHONY: lint +lint: $(GOLANGCI_LINT_BIN) + # Check .golangci.yml for configuration + $(GOLANGCI_LINT_BIN) run -v --enable-all -c .golangci.yml + +.PHONY: fix +fix: $(GOLANGCI_LINT_BIN) format + $(GOLANGCI_LINT_BIN) run --fix --enable-all -c .golangci.yml + +.PHONY: format +format: + @gofmt -w -s $(GO_FILES) -test: - ./test +$(GOTEST_BIN): + GO111MODULE=off go get -u github.com/rakyll/gotest -analyze: - golangci-lint run -v --enable-all -D gochecknoglobals +$(EMBEDMD_BIN): + GO111MODULE=off go get -u github.com/campoy/embedmd -.PHONY: test analyze \ No newline at end of file +$(GOLANGCI_LINT_BIN): + curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/$(GOLANGCI_LINT_VERSION)/install.sh \ + | sed -e '/install -d/d' \ + | sh -s -- -b $(GOPATH)/bin $(GOLANGCI_LINT_VERSION) diff --git a/README.md b/README.md index b0759937..a39b92d5 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ steps: - 'vendor' - name: build - image: golang:1.11-alpine + image: golang:1.13-alpine pull: true commands: - apk add --update make git @@ -87,7 +87,8 @@ steps: ### Using executable (with CLI args) -```console +[embedmd]:# (tmp/help.txt) +```txt NAME: Drone cache plugin - Drone cache plugin @@ -95,12 +96,14 @@ USAGE: drone-cache [global options] command [command options] [arguments...] VERSION: - 1.0.4 + v1.0.4-19-gf16b55a-dirty COMMANDS: help, h Shows a list of commands or help for one command GLOBAL OPTIONS: + --log.level value, --ll value log filtering level. ('error', 'warn', 'info', 'debug') (default: "info") [$PLUGIN_LOG_LEVEL, $ LOG_LEVEL] + --log.format value, --lf value log format to use. ('logfmt', 'json') (default: "logfmt") [$PLUGIN_LOG_FORMAT, $ LOG_FORMAT] --repo.fullname value, --rf value repository full name [$DRONE_REPO] --repo.namespace value, --rns value repository namespace [$DRONE_REPO_NAMESPACE] --repo.owner value, --ro value repository owner (for Drone version < 1.0) [$DRONE_REPO_OWNER] @@ -138,6 +141,8 @@ GLOBAL OPTIONS: --restore, --res restore the cache directories [$PLUGIN_RESTORE] --cache-key value, --chk value cache key to use for the cache directories [$PLUGIN_CACHE_KEY] --archive-format value, --arcfmt value archive format to use to store the cache directories (tar, gzip) (default: "tar") [$PLUGIN_ARCHIVE_FORMAT] + --compression-level value, --cpl value compression level to use for gzip compression when archive-format specified as gzip + (check https://godoc.org/compress/flate#pkg-constants for available options) (default: -1) [$PLUGIN_COMPRESSION_LEVEL] --skip-symlinks, --ss skip symbolic links in archive [$PLUGIN_SKIP_SYMLINKS, $ SKIP_SYMLINKS] --debug, -d debug [$PLUGIN_DEBUG, $ DEBUG] --filesystem-cache-root value, --fcr value local filesystem root directory for the filesystem cache (default: "/tmp/cache") [$PLUGIN_FILESYSTEM_CACHE_ROOT, $ FILESYSTEM_CACHE_ROOT] @@ -148,14 +153,14 @@ GLOBAL OPTIONS: --region value, --reg value AWS bucket region. (us-east-1, eu-west-1, ...) [$PLUGIN_REGION, $S3_REGION] --path-style, --ps use path style for bucket paths. (true for minio, false for aws) [$PLUGIN_PATH_STYLE] --acl value upload files with acl (private, public-read, ...) (default: "private") [$PLUGIN_ACL] - --sftp-cache-root sftp root directory - --sftp-username sftp username - --sftp-password sftp password - --sftp-public-key-file sftp public key file path - --sftp-auth-method sftp auth method (PASSWORD, PUBLIC_KEY_FILE), in case of password use sftp-password else use ftp-public-key-file - --sftp-host sftp host - --sftp-port sftp port --encryption value, --enc value server-side encryption algorithm, defaults to none. (AES256, aws:kms) [$PLUGIN_ENCRYPTION] + --sftp-cache-root value sftp root directory [$SFTP_CACHE_ROOT] + --sftp-username value sftp username [$SFTP_USERNAME] + --sftp-password value sftp password [$SFTP_PASSWORD] + --ftp-public-key-file value sftp public key file path [$SFTP_PUBLIC_KEY_FILE] + --sftp-auth-method value sftp auth method, defaults to none. (PASSWORD, PUBLIC_KEY_FILE) [$SFTP_AUTH_METHOD] + --sftp-host value sftp host [$SFTP_HOST] + --sftp-port value sftp port [$SFTP_PORT] --help, -h show help --version, -v print the version ``` @@ -188,7 +193,7 @@ $ ./scripts/setup_dev_environment.sh ### Tests ```console -$ ./test +$ make test ``` OR @@ -211,7 +216,7 @@ $ go build . Build the docker image with the following commands: ```console -$ make docker-build +$ make container ``` ## Releases diff --git a/cache/backend/backend.go b/cache/backend/backend.go index 413a6bc9..88aae1bf 100644 --- a/cache/backend/backend.go +++ b/cache/backend/backend.go @@ -1,17 +1,19 @@ package backend import ( + "errors" "fmt" "io/ioutil" - "log" "os" "path" "strings" + "github.com/meltwater/drone-cache/cache" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/meltwater/drone-cache/cache" - "github.com/pkg/errors" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" "github.com/pkg/sftp" "golang.org/x/crypto/ssh" ) @@ -52,7 +54,7 @@ type FileSystemConfig struct { } // InitializeS3Backend creates an S3 backend -func InitializeS3Backend(c S3Config, debug bool) (cache.Backend, error) { +func InitializeS3Backend(l log.Logger, c S3Config, debug bool) (cache.Backend, error) { awsConf := &aws.Config{ Region: aws.String(c.Region), Endpoint: &c.Endpoint, @@ -63,11 +65,12 @@ func InitializeS3Backend(c S3Config, debug bool) (cache.Backend, error) { if c.Key != "" && c.Secret != "" { awsConf.Credentials = credentials.NewStaticCredentials(c.Key, c.Secret, "") } else { - log.Println("aws key and/or Secret not provided (falling back to anonymous credentials)") + level.Warn(l).Log("msg", "aws key and/or Secret not provided (falling back to anonymous credentials)") } + level.Debug(l).Log("msg", "s3 backend", "config", fmt.Sprintf("%+v", c)) + if debug { - log.Printf("[DEBUG] s3 backend config: %+v", c) awsConf.WithLogLevel(aws.LogDebugWithHTTPBody) } @@ -75,19 +78,16 @@ func InitializeS3Backend(c S3Config, debug bool) (cache.Backend, error) { } // InitializeFileSystemBackend creates a filesystem backend -func InitializeFileSystemBackend(c FileSystemConfig, debug bool) (cache.Backend, error) { +func InitializeFileSystemBackend(l log.Logger, c FileSystemConfig, debug bool) (cache.Backend, error) { if strings.TrimRight(path.Clean(c.CacheRoot), "/") == "" { - return nil, fmt.Errorf("could not use <%s> as cache root, empty or root path given", c.CacheRoot) + return nil, fmt.Errorf("empty or root path given, <%s> as cache root, ", c.CacheRoot) } if _, err := os.Stat(c.CacheRoot); err != nil { - msg := fmt.Sprintf("could not use <%s> as cache root, make sure volume is mounted", c.CacheRoot) - return nil, errors.Wrap(err, msg) + return nil, fmt.Errorf("make sure volume is mounted, <%s> as cache root %w", c.CacheRoot, err) } - if debug { - log.Printf("[DEBUG] filesystem backend config: %+v", c) - } + level.Debug(l).Log("msg", "filesystem backend", "config", fmt.Sprintf("%+v", c)) return newFileSystem(c.CacheRoot), nil } @@ -114,7 +114,7 @@ type SFTPConfig struct { Auth SSHAuth } -func InitializeSFTPBackend(c SFTPConfig, debug bool) (cache.Backend, error) { +func InitializeSFTPBackend(l log.Logger, c SFTPConfig, debug bool) (cache.Backend, error) { sshClient, err := getSSHClient(c) if err != nil { return nil, err @@ -122,12 +122,10 @@ func InitializeSFTPBackend(c SFTPConfig, debug bool) (cache.Backend, error) { sftpClient, err := sftp.NewClient(sshClient) if err != nil { - return nil, errors.Wrap(err, "unable to connect to ssh with sftp protocol") + return nil, fmt.Errorf("unable to connect to ssh with sftp protocol %w", err) } - if debug { - log.Printf("[DEBUG] sftp backend config: %+v", c) - } + level.Debug(l).Log("msg", "sftp backend", "config", fmt.Sprintf("%+v", c)) return newSftpBackend(sftpClient, c.CacheRoot), nil } @@ -135,7 +133,7 @@ func InitializeSFTPBackend(c SFTPConfig, debug bool) (cache.Backend, error) { func getSSHClient(c SFTPConfig) (*ssh.Client, error) { authMethod, err := getAuthMethod(c) if err != nil { - return nil, errors.Wrap(err, " unable to get ssh auth method") + return nil, fmt.Errorf("unable to get ssh auth method %w", err) } /* #nosec */ @@ -145,7 +143,7 @@ func getSSHClient(c SFTPConfig) (*ssh.Client, error) { HostKeyCallback: ssh.InsecureIgnoreHostKey(), // #nosec just a workaround for now, will fix }) if err != nil { - return nil, errors.Wrap(err, "unable to connect to ssh") + return nil, fmt.Errorf("unable to connect to ssh %w", err) } return client, nil @@ -162,18 +160,20 @@ func getAuthMethod(c SFTPConfig) ([]ssh.AuthMethod, error) { pkAuthMethod, }, err } + return nil, errors.New("ssh method auth is not recognized, should be PASSWORD or PUBLIC_KEY_FILE") } func readPublicKeyFile(file string) (ssh.AuthMethod, error) { buffer, err := ioutil.ReadFile(file) if err != nil { - return nil, errors.Wrap(err, fmt.Sprintf("unable to read file <%s>", file)) + return nil, fmt.Errorf("unable to read file <%s> %w", file, err) } key, err := ssh.ParsePrivateKey(buffer) if err != nil { - return nil, errors.Wrap(err, fmt.Sprintf("unable to parse private key")) + return nil, fmt.Errorf("unable to parse private key %w", err) } + return ssh.PublicKeys(key), nil } diff --git a/cache/backend/filesystem.go b/cache/backend/filesystem.go index 7b329ace..1e833ac4 100644 --- a/cache/backend/filesystem.go +++ b/cache/backend/filesystem.go @@ -6,8 +6,6 @@ import ( "os" "path/filepath" - "github.com/pkg/errors" - "github.com/meltwater/drone-cache/cache" ) @@ -25,7 +23,7 @@ func newFileSystem(cacheRoot string) cache.Backend { func (c *filesystem) Get(p string) (io.ReadCloser, error) { absPath, err := filepath.Abs(filepath.Clean(filepath.Join(c.cacheRoot, p))) if err != nil { - return nil, errors.Wrap(err, "could not get the object") + return nil, fmt.Errorf("get the object %w", err) } return os.Open(absPath) @@ -35,22 +33,22 @@ func (c *filesystem) Get(p string) (io.ReadCloser, error) { func (c *filesystem) Put(p string, src io.ReadSeeker) error { absPath, err := filepath.Abs(filepath.Clean(filepath.Join(c.cacheRoot, p))) if err != nil { - return errors.Wrap(err, "could not build path") + return fmt.Errorf("build path %w", err) } dir := filepath.Dir(absPath) if err := os.MkdirAll(dir, os.FileMode(0755)); err != nil { - return errors.Wrap(err, fmt.Sprintf("could not create directory <%s>", dir)) + return fmt.Errorf("create directory <%s> %w", dir, err) } dst, err := os.Create(absPath) if err != nil { - return errors.Wrap(err, fmt.Sprintf("could not create cache file <%s>", absPath)) + return fmt.Errorf("create cache file <%s> %w", absPath, err) } defer dst.Close() if _, err := io.Copy(dst, src); err != nil { - return errors.Wrap(err, "could not write read seeker as file") + return fmt.Errorf("write read seeker as file %w", err) } return nil diff --git a/cache/backend/s3.go b/cache/backend/s3.go index 19e0efbb..77b41a7d 100644 --- a/cache/backend/s3.go +++ b/cache/backend/s3.go @@ -1,12 +1,12 @@ package backend import ( + "fmt" "io" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" - "github.com/pkg/errors" "github.com/meltwater/drone-cache/cache" ) @@ -22,6 +22,7 @@ type s3Backend struct { // newS3 returns a new S3 remote Backend implemented func newS3(bucket, acl, encryption string, conf *aws.Config) cache.Backend { client := s3.New(session.Must(session.NewSessionWithOptions(session.Options{})), conf) + return &s3Backend{ bucket: bucket, acl: acl, @@ -37,7 +38,7 @@ func (c *s3Backend) Get(p string) (io.ReadCloser, error) { Key: aws.String(p), }) if err != nil { - return nil, errors.Wrap(err, "couldn't get the object") + return nil, fmt.Errorf("get the object %w", err) } return out.Body, nil @@ -54,6 +55,10 @@ func (c *s3Backend) Put(p string, src io.ReadSeeker) error { if c.encryption != "" { in.ServerSideEncryption = aws.String(c.encryption) } - _, err := c.client.PutObject(in) - return errors.Wrap(err, "couldn't put the object") + + if _, err := c.client.PutObject(in); err != nil { + return fmt.Errorf("put the object %w", err) + } + + return nil } diff --git a/cache/backend/sftp.go b/cache/backend/sftp.go index 93cf1bfe..554f501b 100644 --- a/cache/backend/sftp.go +++ b/cache/backend/sftp.go @@ -5,7 +5,6 @@ import ( "io" "path/filepath" - "github.com/pkg/errors" "github.com/pkg/sftp" ) @@ -21,8 +20,9 @@ func newSftpBackend(client *sftp.Client, cacheRoot string) *sftpBackend { func (s sftpBackend) Get(path string) (io.ReadCloser, error) { absPath, err := filepath.Abs(filepath.Clean(filepath.Join(s.cacheRoot, path))) if err != nil { - return nil, errors.Wrap(err, "could not get the object") + return nil, fmt.Errorf("get the object %w", err) } + return s.client.Open(absPath) } @@ -31,17 +31,17 @@ func (s sftpBackend) Put(path string, src io.ReadSeeker) error { dir := filepath.Dir(pathJoin) if err := s.client.MkdirAll(dir); err != nil { - return errors.Wrap(err, fmt.Sprintf("could not create directory <%s>", dir)) + return fmt.Errorf("create directory <%s> %w", dir, err) } dst, err := s.client.Create(pathJoin) if err != nil { - return errors.Wrap(err, fmt.Sprintf("could not create cache file <%s>", pathJoin)) + return fmt.Errorf("create cache file <%s> %w", pathJoin, err) } defer dst.Close() if _, err := io.Copy(dst, src); err != nil { - return errors.Wrap(err, "could not write read seeker as file") + return fmt.Errorf("write read seeker as file %w", err) } return nil diff --git a/cache/backend/sftp_test.go b/cache/backend/sftp_test.go index b9bce26e..08b8aae1 100644 --- a/cache/backend/sftp_test.go +++ b/cache/backend/sftp_test.go @@ -5,6 +5,8 @@ import ( "io/ioutil" "os" "testing" + + "github.com/go-kit/kit/log" ) const defaultSFTPHost = "127.0.0.1" @@ -14,16 +16,17 @@ var host = getEnv("TEST_SFTP_HOST", defaultSFTPHost) var port = getEnv("TEST_SFTP_PORT", defaultSFTPPort) func TestSFTPTruth(t *testing.T) { - cli, err := InitializeSFTPBackend(SFTPConfig{ - CacheRoot: "/upload", - Username: "foo", - Auth: SSHAuth{ - Password: "pass", - Method: SSHAuthMethodPassword, - }, - Host: host, - Port: port, - }, true) + cli, err := InitializeSFTPBackend(log.NewNopLogger(), + SFTPConfig{ + CacheRoot: "/upload", + Username: "foo", + Auth: SSHAuth{ + Password: "pass", + Method: SSHAuthMethodPassword, + }, + Host: host, + Port: port, + }, true) if err != nil { t.Fatal(err) } diff --git a/cache/cache.go b/cache/cache.go index 2b93cfd0..deecd803 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -7,11 +7,11 @@ import ( "fmt" "io" "io/ioutil" - "log" "os" "path/filepath" - "github.com/pkg/errors" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" ) // Backend implements operations for caching files @@ -22,14 +22,28 @@ type Backend interface { // Cache contains configuration for Cache functionality type Cache struct { - skipSymlinks bool - archiveFmt string - b Backend + logger log.Logger + + b Backend + opts options } // New creates a new cache with given parameters -func New(b Backend, archiveFmt string, skipSymlinks bool) Cache { - return Cache{b: b, archiveFmt: archiveFmt, skipSymlinks: skipSymlinks} +func New(logger log.Logger, b Backend, opts ...Option) Cache { + options := options{ + archiveFmt: DefaultArchiveFormat, + compressionLevel: DefaultCompressionLevel, + } + + for _, o := range opts { + o.apply(&options) + } + + return Cache{ + logger: log.With(logger, "component", "cache"), + b: b, + opts: options, + } } // Push pushes the archived file to the cache @@ -37,86 +51,110 @@ func (c Cache) Push(src, dst string) error { // 1. check if source is reachable src, err := filepath.Abs(filepath.Clean(src)) if err != nil { - return errors.Wrap(err, "could not read source directory") + return fmt.Errorf("read source directory %w", err) } - log.Printf("archiving directory <%s>", src) + level.Info(c.logger).Log("msg", "archiving directory", "src", src) // 2. create a temporary file for the archive if err := os.MkdirAll("/tmp", os.FileMode(0755)); err != nil { - return errors.Wrap(err, "could not create tmp directory") + return fmt.Errorf("create tmp directory %w", err) } dir, err := ioutil.TempDir("", "") if err != nil { - return errors.Wrap(err, "could not create tmp folder for archive") + return fmt.Errorf("create tmp folder for archive %w", err) } + archivePath := filepath.Join(dir, "archive.tar") + file, err := os.Create(archivePath) if err != nil { - return errors.Wrap(err, fmt.Sprintf("could not create tarball file <%s>", archivePath)) + return fmt.Errorf("create tarball file <%s> %w", archivePath, err) + } + + tw, twCloser, err := archiveWriter(file, c.opts.archiveFmt, c.opts.compressionLevel) + if err != nil { + return fmt.Errorf("initialize archive writer %w", err) } - tw, twCloser := archiveWriter(file, c.archiveFmt) + + level.Debug(c.logger).Log("msg", "archive compression level", "level", c.opts.compressionLevel) + closer := func() { twCloser() file.Close() } + defer closer() // 3. walk through source and add each file - err = filepath.Walk(src, writeToArchive(tw, c.skipSymlinks)) + err = filepath.Walk(src, writeToArchive(tw, c.opts.skipSymlinks)) if err != nil { - return errors.Wrap(err, "could not add all files to archive") + return fmt.Errorf("add all files to archive %w", err) } // 4. Close resources before upload closer() // 5. upload archive file to server - log.Printf("uploading archived directory <%s> to <%s>", src, dst) + level.Info(c.logger).Log("msg", "uploading archived directory", "src", src, "dst", dst) + return c.pushArchive(dst, archivePath) } func (c Cache) pushArchive(dst, archivePath string) error { f, err := os.Open(archivePath) if err != nil { - return errors.Wrap(err, "could not open archived file to send") + return fmt.Errorf("open archived file to send %w", err) } defer f.Close() - return errors.Wrap(c.b.Put(dst, f), "could not upload file") + if err := c.b.Put(dst, f); err != nil { + return fmt.Errorf("upload file %w", err) + } + + return nil } // Pull fetches the archived file from the cache and restores to the host machine's file system func (c Cache) Pull(src, dst string) error { - log.Printf("downloading archived directory <%s>", src) + level.Info(c.logger).Log("msg", "downloading archived directory", "src", src) // 1. download archive rc, err := c.b.Get(src) if err != nil { - return errors.Wrap(err, "could not get file from storage backend") + return fmt.Errorf("get file from storage backend %w", err) } defer rc.Close() // 2. extract archive - log.Printf("extracting archived directory <%s> to <%s>", src, dst) - tr := archiveReader(rc, c.archiveFmt) - return errors.Wrap(extractFromArchive(tr), "could not extract files from downloaded archive") + level.Info(c.logger).Log("msg", "extracting archived directory", "src", src, "dst", dst) + + if err := extractFromArchive(archiveReader(rc, c.opts.archiveFmt)); err != nil { + return fmt.Errorf("extract files from downloaded archive %w", err) + } + + return nil } // Helpers -func archiveWriter(w io.Writer, archiveFmt string) (*tar.Writer, func()) { - switch archiveFmt { +func archiveWriter(w io.Writer, f string, l int) (*tar.Writer, func(), error) { + switch f { case "gzip": - gw := gzip.NewWriter(w) + gw, err := gzip.NewWriterLevel(w, l) + if err != nil { + return nil, nil, fmt.Errorf("create archive writer %w", err) + } + tw := tar.NewWriter(gw) + return tw, func() { gw.Close() tw.Close() - } + }, nil default: tw := tar.NewWriter(w) - return tw, func() { tw.Close() } + return tw, func() { tw.Close() }, nil } } @@ -131,7 +169,7 @@ func writeToArchive(tw *tar.Writer, skipSymlinks bool) func(path string, fi os.F var err error h, err = tar.FileInfoHeader(fi, "") if err != nil { - return errors.Wrap(err, fmt.Sprintf("could not create header for <%s>", path)) + return fmt.Errorf("create header for <%s> %w", path, err) } if isSymlink(fi) { @@ -141,19 +179,19 @@ func writeToArchive(tw *tar.Writer, skipSymlinks bool) func(path string, fi os.F var err error if h, err = createSymlinkHeader(fi, path); err != nil { - return errors.Wrap(err, "could not create header for symbolic link") + return fmt.Errorf("create header for symbolic link %w", err) } } h.Name = path // to give absolute path if err := tw.WriteHeader(h); err != nil { - return errors.Wrap(err, fmt.Sprintf("could not write header for <%s>", path)) + return fmt.Errorf("write header for <%s> %w", path, err) } if fi.Mode().IsRegular() { // open and write only if it is a regular file if err := writeFileToArchive(tw, path); err != nil { - return errors.Wrap(err, "could not write file to archive") + return fmt.Errorf("write file to archive %w", err) } } @@ -164,12 +202,12 @@ func writeToArchive(tw *tar.Writer, skipSymlinks bool) func(path string, fi os.F func createSymlinkHeader(fi os.FileInfo, path string) (*tar.Header, error) { lnk, err := os.Readlink(path) if err != nil { - return nil, errors.Wrap(err, fmt.Sprintf("could not read link <%s>", path)) + return nil, fmt.Errorf("read link <%s> %w", path, err) } h, err := tar.FileInfoHeader(fi, lnk) if err != nil { - return nil, errors.Wrap(err, fmt.Sprintf("could not create symlink header for <%s>", path)) + return nil, fmt.Errorf("create symlink header for <%s> %w", path, err) } return h, nil @@ -178,12 +216,12 @@ func createSymlinkHeader(fi os.FileInfo, path string) (*tar.Header, error) { func writeFileToArchive(tw io.Writer, path string) error { f, err := os.Open(path) if err != nil { - return errors.Wrap(err, fmt.Sprintf("could not open file <%s>", path)) + return fmt.Errorf("open file <%s> %w", path, err) } defer f.Close() if _, err := io.Copy(tw, f); err != nil { - return errors.Wrap(err, fmt.Sprintf("could not copy the file <%s> data to the tarball", path)) + return fmt.Errorf("copy the file <%s> data to the tarball %w", path, err) } return nil @@ -191,6 +229,7 @@ func writeFileToArchive(tw io.Writer, path string) error { func archiveReader(r io.Reader, archiveFmt string) *tar.Reader { tr := tar.NewReader(r) + switch archiveFmt { case "gzip": gzr, err := gzip.NewReader(r) @@ -198,6 +237,7 @@ func archiveReader(r io.Reader, archiveFmt string) *tar.Reader { gzr.Close() return tr } + return tar.NewReader(gzr) default: return tr @@ -207,11 +247,12 @@ func archiveReader(r io.Reader, archiveFmt string) *tar.Reader { func extractFromArchive(tr *tar.Reader) error { for { h, err := tr.Next() + switch { case err == io.EOF: // if no more files are found return return nil case err != nil: // return any other error - return errors.Wrap(err, "tar reader failed") + return fmt.Errorf("tar reader failed %w", err) case h == nil: // if the header is nil, skip it continue } @@ -221,69 +262,77 @@ func extractFromArchive(tr *tar.Reader) error { if err := extractDir(h); err != nil { return err } + continue case tar.TypeReg, tar.TypeRegA, tar.TypeChar, tar.TypeBlock, tar.TypeFifo: if err := extractRegular(h, tr); err != nil { - return errors.Wrap(err, "could not extract regular file") + return fmt.Errorf("extract regular file %w", err) } + continue case tar.TypeSymlink: if err := extractSymlink(h); err != nil { - return errors.Wrap(err, "could not extract symbolic link") + return fmt.Errorf("extract symbolic link %w", err) } + continue case tar.TypeLink: if err := extractLink(h); err != nil { - return errors.Wrap(err, "could not extract link") + return fmt.Errorf("extract link %w", err) } + continue case tar.TypeXGlobalHeader: continue default: - return fmt.Errorf("could not extract %s, unknown type flag: %c", h.Name, h.Typeflag) + return fmt.Errorf("extract %s, unknown type flag: %c", h.Name, h.Typeflag) } } } func extractDir(h *tar.Header) error { if err := os.MkdirAll(h.Name, os.FileMode(h.Mode)); err != nil { - return errors.Wrap(err, fmt.Sprintf("could not create directory <%s>", h.Name)) + return fmt.Errorf("create directory <%s> %w", h.Name, err) } + return nil } func extractRegular(h *tar.Header, tr io.Reader) error { f, err := os.OpenFile(h.Name, os.O_CREATE|os.O_RDWR, os.FileMode(h.Mode)) if err != nil { - return errors.Wrap(err, fmt.Sprintf("could not open extracted file for writing <%s>", h.Name)) + return fmt.Errorf("open extracted file for writing <%s> %w", h.Name, err) } defer f.Close() if _, err := io.Copy(f, tr); err != nil { - return errors.Wrap(err, fmt.Sprintf("could not copy extracted file for writing <%s>", h.Name)) + return fmt.Errorf("copy extracted file for writing <%s> %w", h.Name, err) } + return nil } func extractSymlink(h *tar.Header) error { if err := unlink(h.Name); err != nil { - return errors.Wrap(err, fmt.Sprintf("could not unlink <%s>", h.Name)) + return fmt.Errorf("unlink <%s> %w", h.Name, err) } if err := os.Symlink(h.Linkname, h.Name); err != nil { - return errors.Wrap(err, fmt.Sprintf("could not create symbolic link <%s>", h.Name)) + return fmt.Errorf("create symbolic link <%s> %w", h.Name, err) } + return nil } func extractLink(h *tar.Header) error { if err := unlink(h.Name); err != nil { - return errors.Wrap(err, fmt.Sprintf("could not unlink <%s>", h.Name)) + return fmt.Errorf("unlink <%s> %w", h.Name, err) } if err := os.Link(h.Linkname, h.Name); err != nil { - return errors.Wrap(err, fmt.Sprintf("could not create hard link <%s>", h.Linkname)) + return fmt.Errorf("create hard link <%s> %w", h.Linkname, err) } + return nil } @@ -296,5 +345,6 @@ func unlink(path string) error { if err == nil { return os.Remove(path) } + return nil } diff --git a/cache/option.go b/cache/option.go new file mode 100644 index 00000000..89a9a783 --- /dev/null +++ b/cache/option.go @@ -0,0 +1,48 @@ +package cache + +import ( + "compress/flate" +) + +const ( + DefaultCompressionLevel = flate.DefaultCompression + DefaultArchiveFormat = "tar" +) + +type options struct { + archiveFmt string + compressionLevel int + skipSymlinks bool +} + +// Option overrides behavior of Cache. +type Option interface { + apply(*options) +} + +type optionFunc func(*options) + +func (f optionFunc) apply(o *options) { + f(o) +} + +// WithSkipSymlinks sets skip symlink option. +func WithSkipSymlinks(b bool) Option { + return optionFunc(func(o *options) { + o.skipSymlinks = b + }) +} + +// WithArchiveFormat sets archive format option. +func WithArchiveFormat(s string) Option { + return optionFunc(func(o *options) { + o.archiveFmt = s + }) +} + +// WithCompressionLevel sets compression level option. +func WithCompressionLevel(i int) Option { + return optionFunc(func(o *options) { + o.compressionLevel = i + }) +} diff --git a/docker/Dockerfile.linux.386 b/docker/Dockerfile.linux.386 index fef574ca..29271c32 100644 --- a/docker/Dockerfile.linux.386 +++ b/docker/Dockerfile.linux.386 @@ -1,4 +1,4 @@ -FROM golang:1.11-alpine AS builder +FROM golang:1.13-alpine AS builder RUN apk add --update --no-cache ca-certificates tzdata && update-ca-certificates RUN echo "[WARNING] Make sure you have run 'goreleaser release', before 'docker build'!" diff --git a/docker/Dockerfile.linux.amd64 b/docker/Dockerfile.linux.amd64 index 69ba54a6..9664ed70 100644 --- a/docker/Dockerfile.linux.amd64 +++ b/docker/Dockerfile.linux.amd64 @@ -1,4 +1,4 @@ -FROM golang:1.11-alpine AS builder +FROM golang:1.13-alpine AS builder RUN apk add --update --no-cache ca-certificates tzdata && update-ca-certificates RUN echo "[WARNING] Make sure you have run 'goreleaser release', before 'docker build'!" diff --git a/docker/Dockerfile.linux.arm64 b/docker/Dockerfile.linux.arm64 index 24393ca8..4f38a52b 100644 --- a/docker/Dockerfile.linux.arm64 +++ b/docker/Dockerfile.linux.arm64 @@ -1,4 +1,4 @@ -FROM golang:1.11-alpine AS builder +FROM golang:1.13-alpine AS builder RUN apk add --update --no-cache ca-certificates tzdata && update-ca-certificates RUN echo "[WARNING] Make sure you have run 'goreleaser release', before 'docker build'!" diff --git a/docker/Dockerfile.linux.arm_5 b/docker/Dockerfile.linux.arm_5 index 97a03a00..1d6a0ba0 100644 --- a/docker/Dockerfile.linux.arm_5 +++ b/docker/Dockerfile.linux.arm_5 @@ -1,4 +1,4 @@ -FROM golang:1.11-alpine AS builder +FROM golang:1.13-alpine AS builder RUN apk add --update --no-cache ca-certificates tzdata && update-ca-certificates RUN echo "[WARNING] Make sure you have run 'goreleaser release', before 'docker build'!" diff --git a/docker/Dockerfile.linux.arm_6 b/docker/Dockerfile.linux.arm_6 index 06fe3855..6d0bdf3c 100644 --- a/docker/Dockerfile.linux.arm_6 +++ b/docker/Dockerfile.linux.arm_6 @@ -1,4 +1,4 @@ -FROM golang:1.11-alpine AS builder +FROM golang:1.13-alpine AS builder RUN apk add --update --no-cache ca-certificates tzdata && update-ca-certificates RUN echo "[WARNING] Make sure you have run 'goreleaser release', before 'docker build'!" diff --git a/docker/Dockerfile.linux.arm_7 b/docker/Dockerfile.linux.arm_7 index 3e995f20..d2647f41 100644 --- a/docker/Dockerfile.linux.arm_7 +++ b/docker/Dockerfile.linux.arm_7 @@ -1,4 +1,4 @@ -FROM golang:1.11-alpine AS builder +FROM golang:1.13-alpine AS builder RUN apk add --update --no-cache ca-certificates tzdata && update-ca-certificates RUN echo "[WARNING] Make sure you have run 'goreleaser release', before 'docker build'!" diff --git a/docs/examples/drone-1.0.md b/docs/examples/drone-1.0.md index 553e78ff..3861dd1c 100644 --- a/docs/examples/drone-1.0.md +++ b/docs/examples/drone-1.0.md @@ -27,7 +27,7 @@ steps: - 'vendor' - name: build - image: golang:1.11-alpine + image: golang:1.13-alpine pull: true commands: - apk add --update make git @@ -74,7 +74,7 @@ steps: path: /tmp/cache - name: build - image: golang:1.11-alpine + image: golang:1.13-alpine pull: true commands: - apk add --update make git @@ -127,7 +127,7 @@ steps: - 'vendor' - name: build - image: golang:1.11-alpine + image: golang:1.13-alpine pull: true commands: - apk add --update make git @@ -175,7 +175,7 @@ steps: - 'vendor' - name: build - image: golang:1.11-alpine + image: golang:1.13-alpine pull: true commands: - apk add --update make git @@ -214,7 +214,7 @@ steps: debug: true - name: build - image: golang:1.11-alpine + image: golang:1.13-alpine pull: true commands: - apk add --update make git diff --git a/go.mod b/go.mod index 886a8832..e4b8d8ec 100644 --- a/go.mod +++ b/go.mod @@ -3,20 +3,21 @@ module github.com/meltwater/drone-cache require ( github.com/aws/aws-sdk-go v1.16.35 github.com/davecgh/go-spew v1.1.1 // indirect - github.com/fatih/color v1.7.0 // indirect github.com/go-ini/ini v1.41.0 // indirect + github.com/go-kit/kit v0.9.0 + github.com/go-logfmt/logfmt v0.4.0 // indirect + github.com/go-stack/stack v1.8.0 // indirect github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect github.com/jtolds/gls v4.2.1+incompatible // indirect - github.com/mattn/go-colorable v0.1.4 // indirect - github.com/mattn/go-isatty v0.0.10 // indirect github.com/minio/minio-go v6.0.14+incompatible github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/pkg/errors v0.8.1 github.com/pkg/sftp v1.10.1 - github.com/rakyll/gotest v0.0.0-20180125184505-86f0749cd8cc // indirect github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 // indirect github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c // indirect github.com/urfave/cli v1.20.0 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 + golang.org/x/sys v0.0.0-20191008105621-543471e840be // indirect gopkg.in/ini.v1 v1.41.0 // indirect ) + +go 1.13 diff --git a/go.sum b/go.sum index 4d495c0e..2a7dabe1 100644 --- a/go.sum +++ b/go.sum @@ -3,10 +3,14 @@ github.com/aws/aws-sdk-go v1.16.35/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpi github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/go-ini/ini v1.41.0 h1:526aoxDtxRHFQKMZfcX2OG9oOI8TJ5yPLM0Mkno/uTY= github.com/go-ini/ini v1.41.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= @@ -15,11 +19,8 @@ github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpR github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -30,8 +31,6 @@ github.com/pkg/sftp v1.10.1 h1:VasscCm72135zRysgrJDKsntdmPN+OuU3+nnHYA9wyc= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rakyll/gotest v0.0.0-20180125184505-86f0749cd8cc h1:hrzpgS8mnUi65ieVrD3TKJMxHP84bzmybMTQIdK/XhM= -github.com/rakyll/gotest v0.0.0-20180125184505-86f0749cd8cc/go.mod h1:iln+RRtJaJ52lKwqrSmNgQYw32Fk16CgChX85eFqBgI= github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 h1:Jpy1PXuP99tXNrhbq2BaPz9B+jNAvH1JPQQpG/9GCXY= github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c h1:Ho+uVpkel/udgjbwB5Lktg9BtvJSh2DT0Hi6LPSyI2w= @@ -47,7 +46,6 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU= diff --git a/internal/logger.go b/internal/logger.go new file mode 100644 index 00000000..13235a8e --- /dev/null +++ b/internal/logger.go @@ -0,0 +1,48 @@ +package internal + +import ( + "os" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" +) + +const ( + LogFormatLogfmt = "logfmt" + LogFormatJSON = "json" + + LogLevelError = "error" + LogLevelWarn = "warn" + LogLevelInfo = "info" + LogLevelDebug = "debug" +) + +func NewLogger(logLevel, logFormat, name string) log.Logger { + var ( + logger log.Logger + lvl level.Option + ) + + switch logLevel { + case LogLevelError: + lvl = level.AllowError() + case LogLevelWarn: + lvl = level.AllowWarn() + case LogLevelInfo: + lvl = level.AllowInfo() + case LogLevelDebug: + lvl = level.AllowDebug() + default: + panic("unexpected log level") + } + + logger = log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)) + if logFormat == LogFormatJSON { + logger = log.NewJSONLogger(log.NewSyncWriter(os.Stderr)) + } + + logger = level.NewFilter(logger, lvl) + logger = log.With(logger, "name", name) + + return log.With(logger, "ts", log.DefaultTimestampUTC, "caller", log.DefaultCaller) +} diff --git a/main.go b/main.go index 30c98071..192b6e94 100644 --- a/main.go +++ b/main.go @@ -1,23 +1,45 @@ package main import ( - "log" + "errors" + stdlog "log" "os" - "github.com/urfave/cli" - + "github.com/meltwater/drone-cache/cache" "github.com/meltwater/drone-cache/cache/backend" + "github.com/meltwater/drone-cache/internal" "github.com/meltwater/drone-cache/metadata" "github.com/meltwater/drone-cache/plugin" + + "github.com/go-kit/kit/log/level" + "github.com/urfave/cli" ) +var version = "0.0.0" + +//nolint:funlen func main() { app := cli.NewApp() app.Name = "Drone cache plugin" app.Usage = "Drone cache plugin" app.Action = run - app.Version = "1.0.4" + app.Version = version app.Flags = []cli.Flag{ + // Logger args + + cli.StringFlag{ + Name: "log.level, ll", + Usage: "log filtering level. ('error', 'warn', 'info', 'debug')", + Value: internal.LogLevelInfo, + EnvVar: "PLUGIN_LOG_LEVEL, LOG_LEVEL", + }, + cli.StringFlag{ + Name: "log.format, lf", + Usage: "log format to use. ('logfmt', 'json')", + Value: internal.LogFormatLogfmt, + EnvVar: "PLUGIN_LOG_FORMAT, LOG_FORMAT", + }, + // Repo args cli.StringFlag{ @@ -221,9 +243,16 @@ func main() { cli.StringFlag{ Name: "archive-format, arcfmt", Usage: "archive format to use to store the cache directories (tar, gzip)", - Value: "tar", + Value: cache.DefaultArchiveFormat, EnvVar: "PLUGIN_ARCHIVE_FORMAT", }, + cli.IntFlag{ + Name: "compression-level, cpl", + Usage: `compression level to use for gzip compression when archive-format specified as gzip + (check https://godoc.org/compress/flate#pkg-constants for available options)`, + Value: cache.DefaultCompressionLevel, + EnvVar: "PLUGIN_COMPRESSION_LEVEL", + }, cli.BoolFlag{ Name: "skip-symlinks, ss", Usage: "skip symbolic links in archive", @@ -332,12 +361,21 @@ func main() { } if err := app.Run(os.Args); err != nil { - log.Fatalf("%+v", err) + stdlog.Fatalf("%+v", err) } } +//nolint:funlen func run(c *cli.Context) error { + var logLevel = c.String("log.level") + if c.Bool("debug") { + logLevel = internal.LogLevelDebug + } + + logger := internal.NewLogger(logLevel, c.String("log.format"), "drone-cache") + plg := plugin.Plugin{ + Logger: logger, Metadata: metadata.Metadata{ Repo: metadata.Repo{ Namespace: c.String("repo.namespace"), @@ -374,13 +412,14 @@ func run(c *cli.Context) error { }, }, Config: plugin.Config{ - ArchiveFormat: c.String("archive-format"), - Backend: c.String("backend"), - CacheKey: c.String("cache-key"), - Debug: c.Bool("debug"), - Mount: c.StringSlice("mount"), - Rebuild: c.Bool("rebuild"), - Restore: c.Bool("restore"), + ArchiveFormat: c.String("archive-format"), + Backend: c.String("backend"), + CacheKey: c.String("cache-key"), + CompressionLevel: c.Int("compression-level"), + Debug: c.Bool("debug"), + Mount: c.StringSlice("mount"), + Rebuild: c.Bool("rebuild"), + Restore: c.Bool("restore"), FileSystem: backend.FileSystemConfig{ CacheRoot: c.String("filesystem-cache-root"), @@ -416,14 +455,16 @@ func run(c *cli.Context) error { } if c.Bool("exit-code") { - // If it is exit-code enabled, always exit with error - log.Println("silent fails disabled, exiting with status code on error") + // If it is exit-code enabled, always exit with error. + level.Warn(logger).Log("msg", "silent fails disabled, exiting with status code on error") return err } - if _, ok := err.(plugin.Error); ok { - // If it is an expected error log it, handle it gracefully - log.Println(err) + var e plugin.Error + if errors.As(err, &e) { + // If it is an expected error log it, handle it gracefully, + level.Error(logger).Log("err", err) + return nil } diff --git a/plugin/cachekey/cachekey.go b/plugin/cachekey/cachekey.go index e226a281..4d6e3286 100644 --- a/plugin/cachekey/cachekey.go +++ b/plugin/cachekey/cachekey.go @@ -2,6 +2,7 @@ package cachekey import ( "crypto/md5" // #nosec + "errors" "fmt" "io" "log" @@ -13,8 +14,6 @@ import ( "text/template" "time" - "github.com/pkg/errors" - "github.com/meltwater/drone-cache/metadata" ) @@ -53,13 +52,14 @@ func Generate(tmpl, mount string, data metadata.Metadata) (string, error) { t, err := ParseTemplate(tmpl) if err != nil { - return "", errors.Wrap(err, fmt.Sprintf("could not parse <%s> as cache key template, falling back to default", tmpl)) + return "", fmt.Errorf("parse, <%s> as cache key template, falling back to default %w", tmpl, err) } var b strings.Builder + err = t.Execute(&b, data) if err != nil { - return "", errors.Wrap(err, fmt.Sprintf("could not build <%s> as cache key, falling back to default. %+v", tmpl, err)) + return "", fmt.Errorf("build, <%s> as cache key, falling back to default %w", tmpl, err) } return filepath.Join(b.String(), mount), nil @@ -76,6 +76,7 @@ func Hash(parts ...string) (string, error) { for i, p := range parts { readers[i] = strings.NewReader(p) } + return readerHasher(readers...) } @@ -84,10 +85,12 @@ func Hash(parts ...string) (string, error) { // readerHasher generic md5 hash generater from io.Readers func readerHasher(readers ...io.Reader) (string, error) { h := md5.New() // #nosec + for _, r := range readers { if _, err := io.Copy(h, r); err != nil { - return "", errors.Wrap(err, "could not write reader as hash") + return "", fmt.Errorf("write reader as hash %w", err) } } + return fmt.Sprintf("%x", h.Sum(nil)), nil } diff --git a/plugin/plugin.go b/plugin/plugin.go index bc7650e0..4d708859 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -2,27 +2,30 @@ package plugin import ( + "errors" "fmt" - "log" "os" "path/filepath" "time" - "github.com/pkg/errors" - "github.com/meltwater/drone-cache/cache" "github.com/meltwater/drone-cache/cache/backend" "github.com/meltwater/drone-cache/metadata" "github.com/meltwater/drone-cache/plugin/cachekey" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" ) type ( - // Config plugin-specific parameters and secrets + // Config plugin-specific parameters and secrets. Config struct { ArchiveFormat string Backend string CacheKey string + CompressionLevel int + Debug bool SkipSymlinks bool Rebuild bool @@ -35,31 +38,33 @@ type ( SFTP backend.SFTPConfig } - // Plugin stores metadata about current plugin + // Plugin stores metadata about current plugin. Plugin struct { + Logger log.Logger Metadata metadata.Metadata Config Config } - // Error recognized error from plugin + // Error recognized error from plugin. Error string ) func (e Error) Error() string { return string(e) } -// Exec entry point of Plugin, where the magic happens +// Exec entry point of Plugin, where the magic happens. func (p *Plugin) Exec() error { c := p.Config // 1. Check parameters if c.Debug { - log.Println("DEBUG MODE enabled!") + level.Debug(p.Logger).Log("msg", "DEBUG MODE enabled!") + for _, pair := range os.Environ() { - log.Println(pair) + level.Debug(p.Logger).Log("var", pair) } - log.Printf("[DEBUG] Plugin initialized with config: %+v", p.Config) - log.Printf("[DEBUG] Plugin initialized with metadata: %+v", p.Metadata) + level.Debug(p.Logger).Log("msg", "plugin initialized with config", "config", fmt.Sprintf("%+v", p.Config)) + level.Debug(p.Logger).Log("msg", "plugin initialized with metadata", "metadata", fmt.Sprintf("%+v", p.Metadata)) } if c.Rebuild && c.Restore { @@ -68,29 +73,32 @@ func (p *Plugin) Exec() error { _, err := cachekey.ParseTemplate(c.CacheKey) if err != nil { - msg := fmt.Sprintf("could not parse <%s> as cache key template, falling back to default", c.CacheKey) - return errors.Wrap(err, msg) + return fmt.Errorf("parse, <%s> as cache key template, falling back to default %w", c.CacheKey, err) } // 2. Initialize backend - backend, err := initializeBackend(c) + backend, err := initializeBackend(p.Logger, c) if err != nil { - return errors.Wrap(err, fmt.Sprintf("could not initialize <%s> as backend", c.Backend)) + return fmt.Errorf("initialize, <%s> as backend %w", c.Backend, err) } // 3. Initialize cache - cch := cache.New(backend, c.ArchiveFormat, c.SkipSymlinks) + cch := cache.New(p.Logger, backend, + cache.WithArchiveFormat(c.ArchiveFormat), + cache.WithSkipSymlinks(c.SkipSymlinks), + cache.WithCompressionLevel(c.CompressionLevel), + ) // 4. Select mode if c.Rebuild { - if err := processRebuild(cch, p.Config.CacheKey, p.Config.Mount, p.Metadata); err != nil { - return Error(fmt.Sprintf("[WARNING] could not build cache, process rebuild failed, %v\n", err)) + if err := processRebuild(p.Logger, cch, p.Config.CacheKey, p.Config.Mount, p.Metadata); err != nil { + return Error(fmt.Sprintf("[WARNING] build cache, process rebuild failed, %v\n", err)) } } if c.Restore { - if err := processRestore(cch, p.Config.CacheKey, p.Config.Mount, p.Metadata); err != nil { - return Error(fmt.Sprintf("[WARNING] could not restore cache, process restore failed, %v\n", err)) + if err := processRestore(p.Logger, cch, p.Config.CacheKey, p.Config.Mount, p.Metadata); err != nil { + return Error(fmt.Sprintf("[WARNING] restore cache, process restore failed, %v\n", err)) } } @@ -98,73 +106,81 @@ func (p *Plugin) Exec() error { } // initializeBackend initializes backend using given configuration -func initializeBackend(c Config) (cache.Backend, error) { +func initializeBackend(logger log.Logger, c Config) (cache.Backend, error) { switch c.Backend { case "s3": - log.Println("[IMPORTANT] using aws s3 as backend") - return backend.InitializeS3Backend(c.S3, c.Debug) + level.Warn(logger).Log("msg", "using aws s3 as backend") + return backend.InitializeS3Backend(logger, c.S3, c.Debug) case "filesystem": - log.Println("[IMPORTANT] using filesystem as backend") - return backend.InitializeFileSystemBackend(c.FileSystem, c.Debug) + level.Warn(logger).Log("msg", "using filesystem as backend") + return backend.InitializeFileSystemBackend(logger, c.FileSystem, c.Debug) case "sftp": - log.Println("[IMPORTANT] using sftp as backend") - return backend.InitializeSFTPBackend(c.SFTP, c.Debug) + level.Warn(logger).Log("msg", "using sftp as backend") + return backend.InitializeSFTPBackend(logger, c.SFTP, c.Debug) default: return nil, errors.New("unknown backend") } } // processRebuild the remote cache from the local environment -func processRebuild(c cache.Cache, cacheKeyTmpl string, mountedDirs []string, m metadata.Metadata) error { +func processRebuild(l log.Logger, c cache.Cache, cacheKeyTmpl string, mountedDirs []string, m metadata.Metadata) error { now := time.Now() branch := m.Commit.Branch for _, mount := range mountedDirs { if _, err := os.Stat(mount); err != nil { - return errors.Wrap(err, fmt.Sprintf("could not mount <%s>, make sure file or directory exists and readable", mount)) + return fmt.Errorf("mount <%s>, make sure file or directory exists and readable %w", mount, err) } - key, err := cacheKey(m, cacheKeyTmpl, mount, branch) + key, err := cacheKey(l, m, cacheKeyTmpl, mount, branch) if err != nil { - return errors.Wrap(err, "could not generate cache key") + return fmt.Errorf("generate cache key %w", err) } + path := filepath.Join(m.Repo.Name, key) - log.Printf("rebuilding cache for directory <%s> to remote cache <%s>", mount, path) + level.Info(l).Log("msg", "rebuilding cache for directory", "local", mount, "remote", path) + if err := c.Push(mount, path); err != nil { - return errors.Wrap(err, "could not upload") + return fmt.Errorf("upload %w", err) } } - log.Printf("cache built in %v", time.Since(now)) + + level.Info(l).Log("msg", "cache built", "took", time.Since(now)) + return nil } // processRestore the local environment from the remote cache -func processRestore(c cache.Cache, cacheKeyTmpl string, mountedDirs []string, m metadata.Metadata) error { +func processRestore(l log.Logger, c cache.Cache, cacheKeyTmpl string, mountedDirs []string, m metadata.Metadata) error { now := time.Now() branch := m.Commit.Branch for _, mount := range mountedDirs { - key, err := cacheKey(m, cacheKeyTmpl, mount, branch) + key, err := cacheKey(l, m, cacheKeyTmpl, mount, branch) if err != nil { - return errors.Wrap(err, "could not generate cache key") + return fmt.Errorf("generate cache key %w", err) } + path := filepath.Join(m.Repo.Name, key) + level.Info(l).Log("msg", "restoring directory", "local", mount, "remote", path) - log.Printf("restoring directory <%s> from remote cache <%s>", mount, path) if err := c.Pull(path, mount); err != nil { - return errors.Wrap(err, "could not download") + return fmt.Errorf("download %w", err) } } - log.Printf("cache restored in %v", time.Since(now)) + + level.Info(l).Log("msg", "cache restored", "took", time.Since(now)) + return nil } // Helpers // cacheKey generates key from given template as parameter or fallbacks hash -func cacheKey(p metadata.Metadata, cacheKeyTmpl, mount, branch string) (string, error) { - log.Println("using provided cache key template") +func cacheKey(l log.Logger, p metadata.Metadata, cacheKeyTmpl, mount, branch string) (string, error) { + level.Info(l).Log("msg", "using provided cache key template") + key, err := cachekey.Generate(cacheKeyTmpl, mount, metadata.Metadata{ Build: p.Build, Commit: p.Commit, @@ -172,10 +188,11 @@ func cacheKey(p metadata.Metadata, cacheKeyTmpl, mount, branch string) (string, }) if err != nil { - log.Printf("%v, falling back to default key", err) + level.Error(l).Log("msg", "falling back to default key", "err", err) key, err = cachekey.Hash(mount, branch) + if err != nil { - return "", errors.Wrap(err, "could not generate hash key for mounted") + return "", fmt.Errorf("generate hash key for mounted %w", err) } } diff --git a/plugin/plugin_test.go b/plugin/plugin_test.go index c3729da1..90b1eff2 100644 --- a/plugin/plugin_test.go +++ b/plugin/plugin_test.go @@ -6,11 +6,12 @@ import ( "path/filepath" "testing" + "github.com/meltwater/drone-cache/cache" "github.com/meltwater/drone-cache/cache/backend" + "github.com/meltwater/drone-cache/metadata" + "github.com/go-kit/kit/log" "github.com/minio/minio-go" - - "github.com/meltwater/drone-cache/metadata" ) const ( @@ -475,6 +476,7 @@ func TestRestoreWithFilesystem(t *testing.T) { func newTestPlugin(bck string, rebuild, restore bool, mount []string, cacheKey, archiveFmt string) Plugin { return Plugin{ + Logger: log.NewNopLogger(), Metadata: metadata.Metadata{ Repo: metadata.Repo{ Branch: "master", @@ -485,12 +487,13 @@ func newTestPlugin(bck string, rebuild, restore bool, mount []string, cacheKey, }, }, Config: Config{ - ArchiveFormat: archiveFmt, - Backend: bck, - CacheKey: cacheKey, - Mount: mount, - Rebuild: rebuild, - Restore: restore, + ArchiveFormat: archiveFmt, + CompressionLevel: cache.DefaultCompressionLevel, + Backend: bck, + CacheKey: cacheKey, + Mount: mount, + Rebuild: rebuild, + Restore: restore, FileSystem: backend.FileSystemConfig{ CacheRoot: "../testcache/cache", @@ -577,7 +580,11 @@ func removeAllObjects(minioClient *minio.Client, bucketName string) error { if !open { return nil } - return fmt.Errorf("remove all objects failed, while fetching %v", err) + if err != nil { + return fmt.Errorf("remove all objects failed, while fetching %v", err) + } + + return nil } } } diff --git a/test b/test deleted file mode 100755 index 19f934b4..00000000 --- a/test +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh - -# Enable strict mode, see: http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail -IFS=$'\n\t' - -echo 'Setting up test environemnt...' -(set -x; docker-compose up -d) -(set -x; mkdir -p ./testcache/cache) - -echo 'Testing...' -# make sure https://github.com/rakyll/gotest installed -(set -x; gotest -cover ./...) - -echo 'Done.' -exit 0