From 8b88665b8cd74a832c7b8a249bcdcbb44e5f095d Mon Sep 17 00:00:00 2001 From: Bastian de Byl Date: Fri, 15 Jul 2022 15:20:37 -0400 Subject: [PATCH] dependabot-1 Major linting fixes, code cleanup --- .errcheck_excludes | 3 ++- .golangci.yml | 9 ++++++-- Makefile | 2 +- archive/archive.go | 6 ++--- archive/gzip/gzip.go | 19 +++++++++++---- archive/gzip/gzip_test.go | 2 +- archive/tar/tar.go | 17 +++++++------- archive/tar/tar_test.go | 2 +- archive/zstd/zstd.go | 22 ++++++++++++------ archive/zstd/zstd_test.go | 2 +- cache/cache.go | 3 +-- cache/flusher.go | 5 ++-- cache/rebuilder.go | 11 ++++----- cache/restorer.go | 7 +++--- go.mod | 3 +-- go.sum | 6 ++--- internal/io.go | 2 +- internal/logger.go | 3 ++- internal/plugin/plugin.go | 9 ++++---- internal/plugin/plugin_test.go | 2 +- key/generator/metadata.go | 22 ++++++++++++++---- key/generator/metadata_test.go | 2 +- main.go | 12 +++++----- storage/backend/azure/azure.go | 11 +++++---- storage/backend/azure/azure_test.go | 2 +- storage/backend/backend.go | 5 ++-- storage/backend/filesystem/filesystem.go | 12 ++++++---- storage/backend/filesystem/filesystem_test.go | 2 +- storage/backend/gcs/gcs.go | 19 ++++++++++----- storage/backend/gcs/gcs_test.go | 2 +- storage/backend/s3/s3.go | 10 ++++---- storage/backend/s3/s3_test.go | 2 +- storage/backend/sftp/sftp.go | 16 +++++++++---- storage/backend/sftp/sftp_test.go | 2 +- storage/storage.go | 23 +++++++++++++++---- 35 files changed, 170 insertions(+), 107 deletions(-) diff --git a/.errcheck_excludes b/.errcheck_excludes index 97b97714..9ae86417 100644 --- a/.errcheck_excludes +++ b/.errcheck_excludes @@ -1 +1,2 @@ -(github.com/go-kit/kit/log.Logger).Log +(github.com/go-kit/log.Logger).Log +b.logger.Log diff --git a/.golangci.yml b/.golangci.yml index 31a4a3b3..f1b8d7e8 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -6,11 +6,17 @@ run: tests: false modules-download-mode: vendor timeout: 5m + skip-dirs: + - test linters: enable-all: true disable: + - exhaustivestruct + - exhaustruct - goerr113 + - ireturn + - maintidx - varnamelen linters-settings: @@ -20,5 +26,4 @@ linters-settings: line-length: 120 funlen: lines: 70 - statements: 40 - + statements: 40 \ No newline at end of file diff --git a/Makefile b/Makefile index 6fbfcc62..0de33d31 100644 --- a/Makefile +++ b/Makefile @@ -132,7 +132,7 @@ test-e2e: $(GOTEST) ; $(info $(M) running test-e2e ) lint: ## Runs golangci-lint analysis lint: # Check .golangci.yml for configuration - $(Q) $(GOLANGCI_LINT) run -v --enable-all --skip-dirs tmp -c .golangci.yml + $(Q) $(GOLANGCI_LINT) run -v --skip-dirs tmp -c .golangci.yml .PHONY: fix fix: ## Runs golangci-lint fix diff --git a/archive/archive.go b/archive/archive.go index 6174a297..8a277688 100644 --- a/archive/archive.go +++ b/archive/archive.go @@ -4,12 +4,11 @@ import ( "compress/flate" "io" + "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/meltwater/drone-cache/archive/gzip" "github.com/meltwater/drone-cache/archive/tar" "github.com/meltwater/drone-cache/archive/zstd" - - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" ) const ( @@ -51,6 +50,7 @@ func FromFormat(logger log.Logger, root string, format string, opts ...Option) A return zstd.New(logger, root, options.skipSymlinks, options.compressionLevel) default: level.Error(logger).Log("msg", "unknown archive format", "format", format) + return tar.New(logger, root, options.skipSymlinks) // DefaultArchiveFormat } } diff --git a/archive/gzip/gzip.go b/archive/gzip/gzip.go index d874e27b..c1cf4c7f 100644 --- a/archive/gzip/gzip.go +++ b/archive/gzip/gzip.go @@ -5,10 +5,9 @@ import ( "fmt" "io" + "github.com/go-kit/log" "github.com/meltwater/drone-cache/archive/tar" "github.com/meltwater/drone-cache/internal" - - "github.com/go-kit/kit/log" ) // Archive implements archive for gzip. @@ -34,17 +33,27 @@ func (a *Archive) Create(srcs []string, w io.Writer) (int64, error) { defer internal.CloseWithErrLogf(a.logger, gw, "gzip writer") - return tar.New(a.logger, a.root, a.skipSymlinks).Create(srcs, gw) + wBytes, err := tar.New(a.logger, a.root, a.skipSymlinks).Create(srcs, gw) + if err != nil { + return 0, fmt.Errorf("writing create archive bytes: %w", err) + } + + return wBytes, nil } // Extract reads content from the given archive reader and restores it to the destination, returns written bytes. func (a *Archive) Extract(dst string, r io.Reader) (int64, error) { gr, err := gzip.NewReader(r) if err != nil { - return 0, err + return 0, fmt.Errorf("create archive extractor: %w", err) } defer internal.CloseWithErrLogf(a.logger, gr, "gzip reader") - return tar.New(a.logger, a.root, a.skipSymlinks).Extract(dst, gr) + eBytes, err := tar.New(a.logger, a.root, a.skipSymlinks).Extract(dst, gr) + if err != nil { + return 0, fmt.Errorf("extracting archive bytes: %w", err) + } + + return eBytes, nil } diff --git a/archive/gzip/gzip_test.go b/archive/gzip/gzip_test.go index 91e81ed3..3903a4db 100644 --- a/archive/gzip/gzip_test.go +++ b/archive/gzip/gzip_test.go @@ -9,7 +9,7 @@ import ( "path/filepath" "testing" - "github.com/go-kit/kit/log" + "github.com/go-kit/log" "github.com/meltwater/drone-cache/archive/tar" "github.com/meltwater/drone-cache/test" diff --git a/archive/tar/tar.go b/archive/tar/tar.go index 1896b1ef..ab5130f5 100644 --- a/archive/tar/tar.go +++ b/archive/tar/tar.go @@ -10,11 +10,10 @@ import ( "strings" "github.com/go-kit/log" - "github.com/meltwater/drone-cache/internal" ) -const defaultDirPermission = 0755 +const defaultDirPermission = 0o755 var ( // ErrSourceNotReachable means that given source is not reachable. @@ -57,7 +56,7 @@ func (a *Archive) Create(srcs []string, w io.Writer) (int64, error) { return written, nil } -// nolint: lll +// nolint: lll, cyclop func writeToArchive(tw *tar.Writer, root string, skipSymlinks bool, written *int64) func(string, os.FileInfo, error) error { return func(path string, fi os.FileInfo, err error) error { if err != nil { @@ -146,7 +145,7 @@ func createSymlinkHeader(fi os.FileInfo, path string) (*tar.Header, error) { return h, nil } -func writeFileToArchive(tw io.Writer, path string) (n int64, err error) { +func writeFileToArchive(tw io.Writer, path string) (int64, error) { f, err := os.Open(path) if err != nil { return 0, fmt.Errorf("open file <%s>, %w", path, err) @@ -163,6 +162,7 @@ func writeFileToArchive(tw io.Writer, path string) (n int64, err error) { } // Extract reads content from the given archive reader and restores it to the destination, returns written bytes. +// nolint: cyclop func (a *Archive) Extract(dst string, r io.Reader) (int64, error) { var ( written int64 @@ -173,7 +173,7 @@ func (a *Archive) Extract(dst string, r io.Reader) (int64, error) { h, err := tr.Next() switch { - case err == io.EOF: // if no more files are found return + case errors.Is(err, io.EOF): // if no more files are found return return written, nil case err != nil: // return any other error return written, fmt.Errorf("tar reader <%v>, %w", err, ErrArchiveNotReadable) @@ -241,7 +241,7 @@ func extractDir(h *tar.Header, target string) error { return nil } -func extractRegular(h *tar.Header, tr io.Reader, target string) (n int64, err error) { +func extractRegular(h *tar.Header, tr io.Reader, target string) (int64, error) { f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(h.Mode)) if err != nil { return 0, fmt.Errorf("open extracted file for writing <%s>, %w", target, err) @@ -282,9 +282,8 @@ func extractLink(h *tar.Header, target string) error { } func unlink(path string) error { - _, err := os.Lstat(path) - if err == nil { - return os.Remove(path) + if _, err := os.Lstat(path); err == nil { + return fmt.Errorf("error with unlinking: %w", os.Remove(path)) } return nil diff --git a/archive/tar/tar_test.go b/archive/tar/tar_test.go index c9a36881..6506c449 100644 --- a/archive/tar/tar_test.go +++ b/archive/tar/tar_test.go @@ -9,7 +9,7 @@ import ( "github.com/meltwater/drone-cache/test" - "github.com/go-kit/kit/log" + "github.com/go-kit/log" ) var ( diff --git a/archive/zstd/zstd.go b/archive/zstd/zstd.go index a92034d0..b28164bc 100644 --- a/archive/zstd/zstd.go +++ b/archive/zstd/zstd.go @@ -4,12 +4,10 @@ import ( "fmt" "io" + "github.com/go-kit/log" "github.com/klauspost/compress/zstd" - "github.com/meltwater/drone-cache/archive/tar" "github.com/meltwater/drone-cache/internal" - - "github.com/go-kit/kit/log" ) // Archive implements archive for zstd. @@ -30,22 +28,32 @@ func New(logger log.Logger, root string, skipSymlinks bool, compressionLevel int func (a *Archive) Create(srcs []string, w io.Writer) (int64, error) { zw, err := zstd.NewWriter(w, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(a.compressionLevel))) if err != nil { - return 0, fmt.Errorf("create archive writer, %w", err) + return 0, fmt.Errorf("zstd create archive writer, %w", err) } defer internal.CloseWithErrLogf(a.logger, zw, "zstd writer") - return tar.New(a.logger, a.root, a.skipSymlinks).Create(srcs, zw) + wBytes, err := tar.New(a.logger, a.root, a.skipSymlinks).Create(srcs, zw) + if err != nil { + return 0, fmt.Errorf("zstd create archive, %w", err) + } + + return wBytes, nil } // Extract reads content from the given archive reader and restores it to the destination, returns written bytes. func (a *Archive) Extract(dst string, r io.Reader) (int64, error) { zr, err := zstd.NewReader(r) if err != nil { - return 0, err + return 0, fmt.Errorf("zstd create extract archive reader, %w", err) } defer internal.CloseWithErrLogf(a.logger, zr.IOReadCloser(), "zstd reader") - return tar.New(a.logger, a.root, a.skipSymlinks).Extract(dst, zr) + eBytes, err := tar.New(a.logger, a.root, a.skipSymlinks).Extract(dst, zr) + if err != nil { + return 0, fmt.Errorf("zstd extract archive, %w", err) + } + + return eBytes, nil } diff --git a/archive/zstd/zstd_test.go b/archive/zstd/zstd_test.go index 22b3a172..b5f0805a 100644 --- a/archive/zstd/zstd_test.go +++ b/archive/zstd/zstd_test.go @@ -8,7 +8,7 @@ import ( "path/filepath" "testing" - "github.com/go-kit/kit/log" + "github.com/go-kit/log" "github.com/meltwater/drone-cache/archive/tar" "github.com/meltwater/drone-cache/test" diff --git a/cache/cache.go b/cache/cache.go index e8a18f04..3694d7b8 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -4,8 +4,7 @@ package cache import ( "time" - "github.com/go-kit/kit/log" - + "github.com/go-kit/log" "github.com/meltwater/drone-cache/archive" "github.com/meltwater/drone-cache/key" "github.com/meltwater/drone-cache/storage" diff --git a/cache/flusher.go b/cache/flusher.go index 458c2e0c..f2d4e0bb 100644 --- a/cache/flusher.go +++ b/cache/flusher.go @@ -4,11 +4,10 @@ import ( "fmt" "time" + "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/meltwater/drone-cache/storage" "github.com/meltwater/drone-cache/storage/backend" - - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" ) type flusher struct { diff --git a/cache/rebuilder.go b/cache/rebuilder.go index e655bb9c..45430306 100644 --- a/cache/rebuilder.go +++ b/cache/rebuilder.go @@ -8,14 +8,13 @@ import ( "sync" "time" + "github.com/dustin/go-humanize" + "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/meltwater/drone-cache/archive" "github.com/meltwater/drone-cache/internal" "github.com/meltwater/drone-cache/key" "github.com/meltwater/drone-cache/storage" - - "github.com/dustin/go-humanize" - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" ) type rebuilder struct { @@ -133,7 +132,7 @@ func (r rebuilder) rebuild(src, dst string) error { level.Error(r.logger).Log("msg", "pr close", "err", err) } - return err + return fmt.Errorf("rebuilder rebuild put file, %w", err) } level.Debug(r.logger).Log( @@ -165,5 +164,5 @@ func (r rebuilder) generateKey(parts ...string) (string, error) { } } - return "", err + return "", fmt.Errorf("rebuilder generate key, %w", err) } diff --git a/cache/restorer.go b/cache/restorer.go index dc5ed709..bf8552a7 100644 --- a/cache/restorer.go +++ b/cache/restorer.go @@ -7,9 +7,8 @@ import ( "sync" "time" - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" - + "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/meltwater/drone-cache/archive" "github.com/meltwater/drone-cache/internal" "github.com/meltwater/drone-cache/key" @@ -134,5 +133,5 @@ func (r restorer) generateKey(parts ...string) (string, error) { } } - return "", err + return "", fmt.Errorf("restorer generate key, %w", err) } diff --git a/go.mod b/go.mod index a0407a44..1b2c85ed 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/Azure/azure-storage-blob-go v0.15.0 github.com/aws/aws-sdk-go v1.44.55 github.com/dustin/go-humanize v1.0.0 - github.com/go-kit/kit v0.12.0 github.com/go-kit/log v0.2.1 github.com/google/go-cmp v0.5.8 github.com/klauspost/compress v1.15.8 @@ -40,7 +39,7 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.opencensus.io v0.23.0 // indirect golang.org/x/net v0.0.0-20220708220712-1185a9018129 // indirect - golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e // indirect + golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index c30d8f39..7f68ebd1 100644 --- a/go.sum +++ b/go.sum @@ -121,8 +121,6 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.12.0 h1:e4o3o3IsBfAKQh5Qbbiqyfu97Ku7jrO/JbohvztANh4= -github.com/go-kit/kit v0.12.0/go.mod h1:lHd+EkCZPIwYItmGDDRdhinkzX2A1sj+M9biaEaizzs= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= @@ -476,8 +474,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e h1:NHvCuwuS43lGnYhten69ZWqi2QOj/CiDNcKbVqwVoew= -golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/internal/io.go b/internal/io.go index 8c4e6b8e..2947161d 100644 --- a/internal/io.go +++ b/internal/io.go @@ -58,5 +58,5 @@ func closeIo(closer io.Closer) error { return nil } - return err + return fmt.Errorf("error closing io.Closer: %w", err) } diff --git a/internal/logger.go b/internal/logger.go index daeab819..c3707c26 100644 --- a/internal/logger.go +++ b/internal/logger.go @@ -43,6 +43,7 @@ func NewLogger(logLevel, logFormat, name string) log.Logger { logger = level.NewFilter(logger, lvl) logger = log.With(logger, "name", name) + logger = log.With(logger, "ts", log.DefaultTimestampUTC, "caller", log.DefaultCaller) - return log.With(logger, "ts", log.DefaultTimestampUTC, "caller", log.DefaultCaller) + return logger } diff --git a/internal/plugin/plugin.go b/internal/plugin/plugin.go index a31600f0..7879d752 100644 --- a/internal/plugin/plugin.go +++ b/internal/plugin/plugin.go @@ -7,6 +7,8 @@ import ( "os" "path/filepath" + "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/meltwater/drone-cache/archive" "github.com/meltwater/drone-cache/cache" "github.com/meltwater/drone-cache/internal/metadata" @@ -14,9 +16,6 @@ import ( keygen "github.com/meltwater/drone-cache/key/generator" "github.com/meltwater/drone-cache/storage" "github.com/meltwater/drone-cache/storage/backend" - - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" ) // Error recognized error from plugin. @@ -42,7 +41,7 @@ func New(logger log.Logger) *Plugin { } // Exec entry point of Plugin, where the magic happens. -func (p *Plugin) Exec() error { // nolint:funlen +func (p *Plugin) Exec() error { // nolint: funlen,cyclop cfg := p.Config // 1. Check parameters @@ -125,6 +124,7 @@ func (p *Plugin) Exec() error { // nolint:funlen if cfg.Rebuild { if err := c.Rebuild(p.Config.Mount); err != nil { level.Debug(p.logger).Log("err", fmt.Sprintf("%+v\n", err)) + return Error(fmt.Sprintf("[IMPORTANT] build cache, %+v\n", err)) } } @@ -132,6 +132,7 @@ func (p *Plugin) Exec() error { // nolint:funlen if cfg.Restore { if err := c.Restore(p.Config.Mount); err != nil { level.Debug(p.logger).Log("err", fmt.Sprintf("%+v\n", err)) + return Error(fmt.Sprintf("[IMPORTANT] restore cache, %+v\n", err)) } } diff --git a/internal/plugin/plugin_test.go b/internal/plugin/plugin_test.go index 6d832a92..0935c245 100644 --- a/internal/plugin/plugin_test.go +++ b/internal/plugin/plugin_test.go @@ -20,7 +20,7 @@ import ( "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" awss3 "github.com/aws/aws-sdk-go/service/s3" - "github.com/go-kit/kit/log" + "github.com/go-kit/log" pkgsftp "github.com/pkg/sftp" "golang.org/x/crypto/ssh" "google.golang.org/api/option" diff --git a/key/generator/metadata.go b/key/generator/metadata.go index 48885917..478d5bfa 100644 --- a/key/generator/metadata.go +++ b/key/generator/metadata.go @@ -11,13 +11,16 @@ import ( "text/template" "time" - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" - + "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/meltwater/drone-cache/internal" "github.com/meltwater/drone-cache/internal/metadata" ) +const ( + EpochNumBase = 10 +) + // Metadata implements a key generator that uses a template engine to generate keys from give metadata. type Metadata struct { logger log.Logger @@ -35,7 +38,7 @@ func NewMetadata(logger log.Logger, tmpl string, data metadata.Metadata) *Metada data: data, funcMap: template.FuncMap{ "checksum": checksumFunc(logger), - "epoch": func() string { return strconv.FormatInt(time.Now().Unix(), 10) }, + "epoch": func() string { return strconv.FormatInt(time.Now().Unix(), EpochNumBase) }, "arch": func() string { return runtime.GOARCH }, "os": func() string { return runtime.GOOS }, }, @@ -69,13 +72,19 @@ func (g *Metadata) Generate(_ ...string) (string, error) { // Check checks if template is parsable. func (g *Metadata) Check() error { _, err := g.parseTemplate() + return err } // Helpers func (g *Metadata) parseTemplate() (*template.Template, error) { - return template.New("cacheKey").Funcs(g.funcMap).Parse(g.tmpl) + tmpl, err := template.New("cacheKey").Funcs(g.funcMap).Parse(g.tmpl) + if err != nil { + return &template.Template{}, fmt.Errorf("parse template failed, %w", err) + } + + return tmpl, nil } func checksumFunc(logger log.Logger) func(string) string { @@ -83,12 +92,14 @@ func checksumFunc(logger log.Logger) func(string) string { path, err := filepath.Abs(filepath.Clean(p)) if err != nil { level.Error(logger).Log("cache key template/checksum could not find file") + return "" } f, err := os.Open(path) if err != nil { level.Error(logger).Log("cache key template/checksum could not open file") + return "" } @@ -97,6 +108,7 @@ func checksumFunc(logger log.Logger) func(string) string { str, err := readerHasher(f) if err != nil { level.Error(logger).Log("cache key template/checksum could not generate hash") + return "" } diff --git a/key/generator/metadata_test.go b/key/generator/metadata_test.go index 71a9d60b..9662bc39 100644 --- a/key/generator/metadata_test.go +++ b/key/generator/metadata_test.go @@ -4,7 +4,7 @@ import ( "testing" "text/template" - "github.com/go-kit/kit/log" + "github.com/go-kit/log" "github.com/meltwater/drone-cache/internal/metadata" "github.com/meltwater/drone-cache/test" ) diff --git a/main.go b/main.go index a948836d..5f0251a6 100644 --- a/main.go +++ b/main.go @@ -2,9 +2,12 @@ package main import ( "errors" + "fmt" stdlog "log" "os" + "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/meltwater/drone-cache/archive" "github.com/meltwater/drone-cache/internal" "github.com/meltwater/drone-cache/internal/metadata" @@ -16,9 +19,6 @@ import ( "github.com/meltwater/drone-cache/storage/backend/gcs" "github.com/meltwater/drone-cache/storage/backend/s3" "github.com/meltwater/drone-cache/storage/backend/sftp" - - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" "github.com/urfave/cli/v2" ) @@ -483,7 +483,7 @@ func main() { // nolint:funlen func run(c *cli.Context) error { - var logLevel = c.String("log.level") + logLevel := c.String("log.level") if c.Bool("debug") { logLevel = internal.LogLevelDebug } @@ -599,7 +599,7 @@ func run(c *cli.Context) error { // If it is exit-code enabled, always exit with error. level.Warn(logger).Log("msg", "silent fails disabled, exiting with status code on error") - return err + return fmt.Errorf("status code exit, %w", err) } var e plugin.Error @@ -610,5 +610,5 @@ func run(c *cli.Context) error { return nil } - return err + return fmt.Errorf("uncaught error, %w", err) } diff --git a/storage/backend/azure/azure.go b/storage/backend/azure/azure.go index 9b53bf59..66289ece 100644 --- a/storage/backend/azure/azure.go +++ b/storage/backend/azure/azure.go @@ -9,9 +9,8 @@ import ( "net/url" "github.com/Azure/azure-storage-blob-go/azblob" - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" - + "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/meltwater/drone-cache/internal" ) @@ -65,6 +64,7 @@ func New(l log.Logger, c Config) (*Backend, error) { _, err = containerURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone) if err != nil { + // nolint: errorlint ret, ok := err.(azblob.StorageError) if !ok { return nil, fmt.Errorf("azure, unexpected error, %w", err) @@ -79,7 +79,7 @@ func New(l log.Logger, c Config) (*Backend, error) { } // Get writes downloaded content to the given writer. -func (b *Backend) Get(ctx context.Context, p string, w io.Writer) (err error) { +func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { errCh := make(chan error) go func() { @@ -87,9 +87,11 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) (err error) { blobURL := b.containerURL.NewBlockBlobURL(p) + // nolint: lll resp, err := blobURL.Download(ctx, 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{}) if err != nil { errCh <- fmt.Errorf("get the object, %w", err) + return } @@ -106,6 +108,7 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) (err error) { case err := <-errCh: return err case <-ctx.Done(): + // nolint: wrapcheck return ctx.Err() } } diff --git a/storage/backend/azure/azure_test.go b/storage/backend/azure/azure_test.go index 7983a35f..50c51cc1 100644 --- a/storage/backend/azure/azure_test.go +++ b/storage/backend/azure/azure_test.go @@ -11,7 +11,7 @@ import ( "testing" "time" - "github.com/go-kit/kit/log" + "github.com/go-kit/log" "github.com/meltwater/drone-cache/test" ) diff --git a/storage/backend/backend.go b/storage/backend/backend.go index 3f6d622a..ebe1b98d 100644 --- a/storage/backend/backend.go +++ b/storage/backend/backend.go @@ -7,9 +7,8 @@ import ( "io" "time" - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" - + "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/meltwater/drone-cache/storage/backend/azure" "github.com/meltwater/drone-cache/storage/backend/filesystem" "github.com/meltwater/drone-cache/storage/backend/gcs" diff --git a/storage/backend/filesystem/filesystem.go b/storage/backend/filesystem/filesystem.go index 8552f9ee..19debf86 100644 --- a/storage/backend/filesystem/filesystem.go +++ b/storage/backend/filesystem/filesystem.go @@ -9,13 +9,12 @@ import ( "path/filepath" "strings" - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" - + "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/meltwater/drone-cache/internal" ) -const defaultFileMode = 0755 +const defaultFileMode = 0o755 // Backend is an file system implementation of the Backend. type Backend struct { @@ -54,6 +53,7 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { rc, err := os.Open(path) if err != nil { errCh <- fmt.Errorf("get the object, %w", err) + return } @@ -62,6 +62,7 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { _, err = io.Copy(w, rc) if err != nil { errCh <- fmt.Errorf("copy the object, %w", err) + return } }() @@ -70,6 +71,7 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { case err := <-errCh: return err case <-ctx.Done(): + // nolint: wrapcheck return ctx.Err() } } @@ -94,6 +96,7 @@ func (b *Backend) Put(ctx context.Context, p string, r io.Reader) error { w, err := os.Create(path) if err != nil { errCh <- fmt.Errorf("create cache file, %w", err) + return } @@ -112,6 +115,7 @@ func (b *Backend) Put(ctx context.Context, p string, r io.Reader) error { case err := <-errCh: return err case <-ctx.Done(): + // nolint: wrapcheck return ctx.Err() } } diff --git a/storage/backend/filesystem/filesystem_test.go b/storage/backend/filesystem/filesystem_test.go index fec22939..16602e04 100644 --- a/storage/backend/filesystem/filesystem_test.go +++ b/storage/backend/filesystem/filesystem_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "github.com/go-kit/kit/log" + "github.com/go-kit/log" "github.com/meltwater/drone-cache/test" ) diff --git a/storage/backend/gcs/gcs.go b/storage/backend/gcs/gcs.go index c3e86f5a..f48257dc 100644 --- a/storage/backend/gcs/gcs.go +++ b/storage/backend/gcs/gcs.go @@ -3,16 +3,16 @@ package gcs import ( "context" "crypto/tls" + "errors" "fmt" "io" "net/http" "strings" - "github.com/meltwater/drone-cache/internal" - gcstorage "cloud.google.com/go/storage" - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/meltwater/drone-cache/internal" "golang.org/x/oauth2/google" "google.golang.org/api/option" ) @@ -80,6 +80,7 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { r, err := obj.NewReader(ctx) if err != nil { errCh <- fmt.Errorf("get the object, %w", err) + return } @@ -95,6 +96,7 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { case err := <-errCh: return err case <-ctx.Done(): + // nolint: wrapcheck return ctx.Err() } } @@ -136,6 +138,7 @@ func (b *Backend) Put(ctx context.Context, p string, r io.Reader) error { case err := <-errCh: return err case <-ctx.Done(): + // nolint: wrapcheck return ctx.Err() } } @@ -160,8 +163,9 @@ func (b *Backend) Exists(ctx context.Context, p string) (bool, error) { } attrs, err := obj.Attrs(ctx) - if err != nil && err != gcstorage.ErrObjectNotExist { + if err != nil && !errors.Is(err, gcstorage.ErrObjectNotExist) { resCh <- &result{err: fmt.Errorf("get the object attrs, %w", err)} + return } @@ -172,6 +176,7 @@ func (b *Backend) Exists(ctx context.Context, p string) (bool, error) { case res := <-resCh: return res.val, res.err case <-ctx.Done(): + // nolint: wrapcheck return false, ctx.Err() } } @@ -181,12 +186,14 @@ func (b *Backend) Exists(ctx context.Context, p string) (bool, error) { func setAuthenticationMethod(l log.Logger, c Config, opts []option.ClientOption) []option.ClientOption { if c.APIKey != "" { opts = append(opts, option.WithAPIKey(c.APIKey)) + return opts } creds, err := credentials(l, c) if err == nil { opts = append(opts, option.WithCredentials(creds)) + return opts } @@ -211,7 +218,7 @@ func credentials(l log.Logger, c Config) (*google.Credentials, error) { creds, err = google.FindDefaultCredentials(ctx, gcstorage.ScopeFullControl) if err != nil { - return nil, err + return nil, fmt.Errorf("gcs credentials not found, %w", err) } return creds, nil diff --git a/storage/backend/gcs/gcs_test.go b/storage/backend/gcs/gcs_test.go index a7b02e0c..aec9c7e7 100644 --- a/storage/backend/gcs/gcs_test.go +++ b/storage/backend/gcs/gcs_test.go @@ -14,7 +14,7 @@ import ( "time" gcstorage "cloud.google.com/go/storage" - "github.com/go-kit/kit/log" + "github.com/go-kit/log" "github.com/meltwater/drone-cache/test" "google.golang.org/api/option" ) diff --git a/storage/backend/s3/s3.go b/storage/backend/s3/s3.go index 7ed3c53e..6d5140ed 100644 --- a/storage/backend/s3/s3.go +++ b/storage/backend/s3/s3.go @@ -13,9 +13,8 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" - + "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/meltwater/drone-cache/internal" ) @@ -98,6 +97,7 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { out, err := b.client.GetObjectWithContext(ctx, in) if err != nil { errCh <- fmt.Errorf("get the object, %w", err) + return } @@ -113,6 +113,7 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { case err := <-errCh: return err case <-ctx.Done(): + // nolint: wrapcheck return ctx.Err() } } @@ -149,6 +150,7 @@ func (b *Backend) Exists(ctx context.Context, p string) (bool, error) { out, err := b.client.HeadObjectWithContext(ctx, in) if err != nil { + // nolint: errorlint if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == s3.ErrCodeNoSuchKey || awsErr.Code() == "NotFound" { return false, nil } @@ -169,7 +171,6 @@ func assumeRole(l log.Logger, c *aws.Config, roleArn string) credentials.Value { DisableSSL: c.DisableSSL, CredentialsChainVerboseErrors: aws.Bool(true), }) - if err != nil { level.Error(l).Log("msg", "s3 backend", "assume-role", err.Error()) } @@ -177,7 +178,6 @@ func assumeRole(l log.Logger, c *aws.Config, roleArn string) credentials.Value { creds, err := stscreds.NewCredentials(sess, roleArn, func(p *stscreds.AssumeRoleProvider) { p.RoleSessionName = "drone-cache" }).Get() - if err != nil { level.Error(l).Log("msg", "s3 backend", "assume-role", err.Error()) } diff --git a/storage/backend/s3/s3_test.go b/storage/backend/s3/s3_test.go index b2e183e1..668078e8 100644 --- a/storage/backend/s3/s3_test.go +++ b/storage/backend/s3/s3_test.go @@ -14,7 +14,7 @@ import ( "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" - "github.com/go-kit/kit/log" + "github.com/go-kit/log" "github.com/meltwater/drone-cache/test" ) diff --git a/storage/backend/sftp/sftp.go b/storage/backend/sftp/sftp.go index 48c38c31..d0fd3559 100644 --- a/storage/backend/sftp/sftp.go +++ b/storage/backend/sftp/sftp.go @@ -9,12 +9,11 @@ import ( "os" "path/filepath" - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/meltwater/drone-cache/internal" "github.com/pkg/sftp" "golang.org/x/crypto/ssh" - - "github.com/meltwater/drone-cache/internal" ) // Backend implements storage.Backend for sFTP. @@ -46,6 +45,7 @@ func New(l log.Logger, c Config) (*Backend, error) { client, err := sftp.NewClient(sshClient) if err != nil { sshClient.Close() + return nil, fmt.Errorf("unable to connect to ssh with sftp protocol, %w", err) } @@ -73,6 +73,7 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { rc, err := b.client.Open(path) if err != nil { errCh <- fmt.Errorf("get the object, %w", err) + return } @@ -88,6 +89,7 @@ func (b *Backend) Get(ctx context.Context, p string, w io.Writer) error { case err := <-errCh: return err case <-ctx.Done(): + // nolint: wrapcheck return ctx.Err() } } @@ -104,12 +106,14 @@ func (b *Backend) Put(ctx context.Context, p string, r io.Reader) error { dir := filepath.Dir(path) if err := b.client.MkdirAll(dir); err != nil { errCh <- fmt.Errorf("create directory, %w", err) + return } w, err := b.client.Create(path) if err != nil { errCh <- fmt.Errorf("create cache file, %w", err) + return } @@ -128,6 +132,7 @@ func (b *Backend) Put(ctx context.Context, p string, r io.Reader) error { case err := <-errCh: return err case <-ctx.Done(): + // nolint: wrapcheck return ctx.Err() } } @@ -152,6 +157,7 @@ func (b *Backend) Exists(ctx context.Context, p string) (bool, error) { _, err := b.client.Stat(path) if err != nil && !os.IsNotExist(err) { resCh <- &result{err: fmt.Errorf("check the object exists, %w", err)} + return } resCh <- &result{val: err == nil} @@ -161,6 +167,7 @@ func (b *Backend) Exists(ctx context.Context, p string) (bool, error) { case res := <-resCh: return res.val, res.err case <-ctx.Done(): + // nolint: wrapcheck return false, ctx.Err() } } @@ -173,6 +180,7 @@ func authMethod(c Config) ([]ssh.AuthMethod, error) { return []ssh.AuthMethod{ssh.Password(c.Auth.Password)}, nil case SSHAuthMethodPublicKeyFile: pkAuthMethod, err := readPublicKeyFile(c.Auth.PublicKeyFile) + return []ssh.AuthMethod{pkAuthMethod}, err default: return nil, errors.New("unknown ssh method (PASSWORD, PUBLIC_KEY_FILE)") diff --git a/storage/backend/sftp/sftp_test.go b/storage/backend/sftp/sftp_test.go index cb4a1cf2..6f6b16d0 100644 --- a/storage/backend/sftp/sftp_test.go +++ b/storage/backend/sftp/sftp_test.go @@ -12,7 +12,7 @@ import ( "github.com/meltwater/drone-cache/test" - "github.com/go-kit/kit/log" + "github.com/go-kit/log" ) const ( diff --git a/storage/storage.go b/storage/storage.go index 31d2ca81..358697f3 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -2,12 +2,12 @@ package storage import ( "context" + "fmt" "io" "time" + "github.com/go-kit/log" "github.com/meltwater/drone-cache/storage/backend" - - "github.com/go-kit/kit/log" ) const DefaultOperationTimeout = 3 * time.Minute @@ -48,7 +48,11 @@ func (s *storage) Get(p string, w io.Writer) error { ctx, cancel := context.WithTimeout(context.Background(), s.timeout) defer cancel() - return s.b.Get(ctx, p, w) + if err := s.b.Get(ctx, p, w); err != nil { + return fmt.Errorf("storage backend get failure, %w", err) + } + + return nil } // Put writes contents of io.Reader to remote storage at given key location. @@ -56,7 +60,11 @@ func (s *storage) Put(p string, r io.Reader) error { ctx, cancel := context.WithTimeout(context.Background(), s.timeout) defer cancel() - return s.b.Put(ctx, p, r) + if err := s.b.Put(ctx, p, r); err != nil { + return fmt.Errorf("storage backend put failure, %w", err) + } + + return nil } // Exists checks if object with given key exists in remote storage. @@ -64,7 +72,12 @@ func (s *storage) Exists(p string) (bool, error) { ctx, cancel := context.WithTimeout(context.Background(), s.timeout) defer cancel() - return s.b.Exists(ctx, p) + ret, err := s.b.Exists(ctx, p) + if err != nil { + return ret, fmt.Errorf("storage backend exists failure, %w", err) + } + + return ret, nil } // List lists contents of the given directory by given key from remote storage.