Update s3-proxy to use env vars for all secrets

.github/workflows/lint-test.yaml

@@ -26,13 +26,13 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v2
         with:
-          go-version: '1.18'
+          go-version: '1.21'
       - name: Run golangci-lint
         uses: golangci/golangci-lint-action@v2
 
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.45
+          version: v1.59
 
           # Optional: working directory, useful for monorepos
           # working-directory: somedir

.golangci.yml

@@ -1,4 +1,6 @@
 linters-settings:
+  errcheck:
+    ignore: ''
   goimports:
     local-prefixes: github.com/packethost/aws-s3-proxy
   gofumpt:
@@ -7,36 +9,29 @@ linters-settings:
 linters:
   enable:
     # default linters
-    - deadcode
     - errcheck
     - gosimple
     - govet
     - ineffassign
-    - staticcheck
-    - structcheck
     - typecheck
     - unused
-    - varcheck
 
     # additional linters
     - bodyclose
     - gocritic
     - gocyclo
-    - goerr113
+    - err113
     - gofmt
-    # - gofumpt
+    - gofumpt
     - goimports
-    - golint
-    - gomnd
+    - mnd
     - govet
-    - maligned
     - misspell
     - noctx
     - stylecheck
     - whitespace
     - wsl
 
-    # - bod
 issues:
   exclude:
     # Default excludes from `golangci-lint run --help` with EXC0002 removed

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.18-alpine AS builder
+FROM golang:1.21-alpine AS builder
 WORKDIR /root
 
 RUN apk --no-cache add gcc musl-dev git

README.md

@@ -19,11 +19,10 @@ If auth is enable, you can upload to the bucket (assuming your AWS credentials p
 
 Environment Variables     | Description                                       | Required | Default
 ------------------------- | ------------------------------------------------- | -------- | -----------------
-AWS_REGION                | The AWS `region` where the S3 bucket exists.      |          | us-east-1
-AWS_ACCESS_KEY_ID         | AWS `access key` for API access.                  |          | EC2 Instance Role
-AWS_SECRET_ACCESS_KEY     | AWS `secret key` for API access.                  |          | EC2 Instance Role
-AWS_API_ENDPOINT          | The endpoint for AWS API for local development.   |          | -
-S3_PROXY_BASIC_AUTH_PASS  | Password for basic authentication.                |          | -
+PRIMARY_STORE_ACCESS_KEY         | Primary AWS `access key` for API access.                  |          | EC2 Instance Role
+PRIMARY_STORE_SECRET_KEY     | Primary AWS `secret key` for API access.                  |          | EC2 Instance Role
+SECONDARY_STORE_ACCESS_KEY         | Secondary AWS `access key` for API access.                  |          | EC2 Instance Role
+SECONDARY_STORE_SECRET_KEY     | Secondary AWS `secret key` for API access.                  |          | EC2 Instance Role
 
 Other environment variables can be set by `S3_PROXY_` and uppercase CLI options without hyphens or underscores, so `--listen-port` becomes `S3_PROXY_LISTENPORT`.
 
@@ -37,37 +36,34 @@ Usage:
   aws-s3-proxy serve [flags]
 
 Flags:
-      --access-log                         toggle access log
-      --aws-api-endpoint string            AWS API Endpoint
-      --aws-region string                  AWS region for s3, default AWS env vars will override (default "us-east-1")
-      --basic-auth-user string             username for basic auth
-      --content-encoding                   toggle content encoding (default true)
-      --cors-allow-headers string          CORS: Comma-delimited list of the supported request headers
-      --cors-allow-methods string          CORS: comma-delimited list of the allowed - https://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html
-      --cors-allow-origin string           CORS: a URI that may access the resource
-      --cors-max-age int                   CORS: max age in seconds (default 600)
-      --directory-listing                  toggle directory listing
-      --directory-listing-format           toggle directory listing spider formatted
-      --disable-compression                toggle compression (default true)
-      --disable-upstream-ssl               toggle tls for the aws-sdk
-      --enable-upload                      toggle upload, requires auth
-      --get-all-pages-in-dir               toggle getting all pages in directories
-      --guess-bucket-timeout int           timeout, in seconds, for guessing bucket region (default 10)
-      --healthcheck-path string            path for healthcheck
-  -h, --help                               help for serve
-      --http-cache-control Cache-Control   overrides S3's HTTP Cache-Control header
-      --http-expires Expires               overrides S3's HTTP Expires header
-      --idle-connection-timeout int        idle connection timeout in seconds (default 10)
-      --index-document string              the index document for static website (default "index.html")
-      --insecure-tls                       toggle insecure tls
-      --listen-address string              host address to listen on (default "::1")
-      --listen-port string                 port to listen on (default "21080")
-      --max-idle-connections int           max idle connections (default 150)
-      --ssl-cert-path string               path to ssl cert
-      --ssl-key-path string                path to ssl key
-      --strip-path string                  strip path prefix
-      --upstream-bucket string             upstream s3 bucket
-      --upstream-key-prefix string         upstream s3 path/key prefix
+      --facility string                               Location where the service is running
+      --healthcheck-path string                       path for healthcheck
+  -h, --help                                          help for serve
+      --http-cache-control Cache-Control              overrides S3's HTTP Cache-Control header
+      --http-expires Expires                          overrides S3's HTTP Expires header
+      --listen-address string                         host address to listen on (default "::1")
+      --listen-port string                            port to listen on (default "21080")
+      --primary-store-access-key string               s3 access-key
+      --primary-store-bucket string                   bucket name
+      --primary-store-disable-bucket-ssl              toggle tls for the aws-sdk
+      --primary-store-disable-compression             toggle compressions
+      --primary-store-endpoint string                 endpoint URL (hostname only or fully qualified URI)
+      --primary-store-idle-connection-timeout int     idle connection timeout in seconds (default 10)
+      --primary-store-insecure-tls                    toogle tls verify
+      --primary-store-max-idle-connections int        max idle connections (default 150)
+      --primary-store-region string                   region for bucket
+      --primary-store-secret-key string               s3 secret-access-key
+      --secondary-fall-back                           toggle read from secondary
+      --secondary-store-access-key string             s3 access-key
+      --secondary-store-bucket string                 bucket name
+      --secondary-store-disable-bucket-ssl            toggle tls for the aws-sdk
+      --secondary-store-disable-compression           toggle compressions
+      --secondary-store-endpoint string               endpoint URL (hostname only or fully qualified URI)
+      --secondary-store-idle-connection-timeout int   idle connection timeout in seconds (default 10)
+      --secondary-store-insecure-tls                  toogle tls verify
+      --secondary-store-max-idle-connections int      max idle connections (default 150)
+      --secondary-store-region string                 region for bucket
+      --secondary-store-secret-key string             s3 secret-access-key
 
 Global Flags:
       --config string   config file (default is $HOME/.s3-proxy.yaml)

cmd/aws-s3-proxy/pkg/root.go

@@ -2,9 +2,7 @@
 package cmd
 
 import (
-	"fmt"
 	"log"
-	"os"
 	"strings"
 
 	homedir "github.com/mitchellh/go-homedir"
@@ -38,21 +36,22 @@ func init() {
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
 	if err := rootCmd.Execute(); err != nil {
-		log.Fatalf(err.Error())
+		log.Fatal(err)
 	}
 }
 
 // initConfig reads in config file and ENV variables if set.
 func initConfig() {
+	setupLogging()
+
 	if cfgFile != "" {
 		// Use config file from the flag.
 		viper.SetConfigFile(cfgFile)
 	} else {
 		// Find home directory.
 		home, err := homedir.Dir()
 		if err != nil {
-			fmt.Println(err)
-			os.Exit(1)
+			logger.Fatalf("unable to find home directory: %v", err)
 		}
 
 		// Search config in home directory with name ".s3-proxy" (without extension).
@@ -68,17 +67,21 @@ func initConfig() {
 
 	// If a config file is found, read it in.
 	if err := viper.ReadInConfig(); err == nil {
-		fmt.Println("Using config file:", viper.ConfigFileUsed())
+		logger.Infof("using config file: %s", viper.ConfigFileUsed())
 	}
-
-	setupLogging()
 }
 
-// viperBindFlag provides a wrapper around the viper bindings that handles error checks
+// viperBindFlag provides a wrapper around the viper pflag bindings that handles error checks
 func viperBindFlag(name string, flag *pflag.Flag) {
-	err := viper.BindPFlag(name, flag)
-	if err != nil {
-		panic(err)
+	if err := viper.BindPFlag(name, flag); err != nil {
+		logger.Fatalf("failed to bind flag: %v", err)
+	}
+}
+
+// viperBindEnv provides a wrapper around the viper env var bindings that handles error checks
+func viperBindEnv(input ...string) {
+	if err := viper.BindEnv(input...); err != nil {
+		logger.Fatalf("failed to bind environment variable: %v", err)
 	}
 }
 

cmd/aws-s3-proxy/pkg/serve.go

@@ -44,10 +44,10 @@ func httpFlags() {
 	serveCmd.Flags().String("facility", "", "Location where the service is running")
 	viperBindFlag("httpopts.facility", serveCmd.Flags().Lookup("facility"))
 
-	serveCmd.Flags().String("http-cache-control", "", "overrides S3's HTTP `Cache-Control` header")
+	serveCmd.Flags().String("http-cache-control", "", "override S3 HTTP `Cache-Control` header")
 	viperBindFlag("httpopts.httpcachecontrol", serveCmd.Flags().Lookup("http-cache-control"))
 
-	serveCmd.Flags().String("http-expires", "", "overrides S3's HTTP `Expires` header")
+	serveCmd.Flags().String("http-expires", "", "override S3 HTTP `Expires` header")
 	viperBindFlag("httpopts.httpexpires", serveCmd.Flags().Lookup("http-expires"))
 
 	serveCmd.Flags().String("healthcheck-path", "", "path for healthcheck")
@@ -131,6 +131,12 @@ func s3Flags() {
 	}
 
 	for _, store := range stores {
+		envVarAccessKey := strings.ToUpper(strings.ReplaceAll(store, "-", "_")) + "_ACCESS_KEY"
+		envVarSecretKey := strings.ToUpper(strings.ReplaceAll(store, "-", "_")) + "_SECRET_KEY"
+
+		viperBindEnv(titleCase(store)+".AccessKey", envVarAccessKey)
+		viperBindEnv(titleCase(store)+".SecretKey", envVarSecretKey)
+
 		for _, boolFlag := range boolFlags {
 			// concatenated flag name
 			f := fmt.Sprintf("%s-%s", store, boolFlag.long)
@@ -211,6 +217,10 @@ func init() {
 	setupMetrics()
 }
 
+func titleCase(input string) string {
+	return strings.ReplaceAll(strings.ToTitle(strings.ReplaceAll(input, "-", " ")), " ", "")
+}
+
 func setupMetrics() {
 	metricsMW = promMW.Prometheus()
 
@@ -275,13 +285,15 @@ func serve(ctx context.Context) {
 			shutdown <- os.Interrupt
 		}
 
-		logger.Infof("[config] Primary bucket '%s'", config.Cfg.PrimaryStore.Bucket)
+		logger.Infof("[config] primary bucket: Name: %s", config.Cfg.PrimaryStore.Bucket)
+		logger.Debugf("[config] primary bucket details: %s", config.Cfg.PrimaryStore)
 
 		if config.Cfg.ReadThrough.Enabled {
-			logger.Infof("[config] Secondary bucket '%s'", config.Cfg.SecondaryStore.Bucket)
+			logger.Infof("[config] secondary bucket: Name: %s", config.Cfg.SecondaryStore.Bucket)
+			logger.Debugf("[config] primary bucket details: %s", config.Cfg.SecondaryStore)
 
 			if config.Cfg.SecondaryStore.Session == nil {
-				logger.Error("invalid secoindary bucket session")
+				logger.Error("invalid secondary bucket session")
 
 				shutdown <- os.Interrupt
 			}
@@ -301,6 +313,6 @@ func serve(ctx context.Context) {
 	}()
 
 	if err := router.Shutdown(ctx); err != nil {
-		logger.Errorf("Failed graceful shutdown", err)
+		logger.Errorf("failed graceful shutdown", err)
 	}
 }

go.mod

@@ -1,60 +1,58 @@
 module github.com/packethost/aws-s3-proxy
 
-go 1.18
+go 1.21
+
+toolchain go1.22.3
 
 require (
-	github.com/aws/aws-sdk-go v1.43.8
-	github.com/labstack/echo-contrib v0.12.0
-	github.com/labstack/echo/v4 v4.6.3
+	github.com/aws/aws-sdk-go v1.53.13
+	github.com/labstack/echo-contrib v0.17.1
+	github.com/labstack/echo/v4 v4.12.0
 	github.com/mitchellh/go-homedir v1.1.0
-	github.com/prometheus/client_golang v1.12.1
-	github.com/spf13/cobra v1.3.0
+	github.com/prometheus/client_golang v1.19.1
+	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
-	github.com/spf13/viper v1.10.1
-	github.com/stretchr/testify v1.7.0
-	go.uber.org/automaxprocs v1.4.0
-	go.uber.org/zap v1.21.0
+	github.com/spf13/viper v1.18.2
+	github.com/stretchr/testify v1.9.0
+	go.uber.org/automaxprocs v1.5.3
+	go.uber.org/zap v1.27.0
 )
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-cmp v0.5.7 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/kr/text v0.2.0 // indirect
-	github.com/labstack/gommon v0.3.1 // indirect
-	github.com/magiconair/properties v1.8.6 // indirect
-	github.com/mattn/go-colorable v0.1.12 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
-	github.com/mitchellh/mapstructure v1.4.3 // indirect
-	github.com/pelletier/go-toml v1.9.4 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.32.1 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/spf13/afero v1.8.1 // indirect
-	github.com/spf13/cast v1.4.1 // indirect
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/labstack/gommon v0.4.2 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.53.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/sagikazarmark/locafero v0.4.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.6.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
-	github.com/valyala/fasttemplate v1.2.1 // indirect
-	go.uber.org/atomic v1.9.0 // indirect
-	go.uber.org/multierr v1.7.0 // indirect
-	golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
-	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
-	golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
-	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
-	gopkg.in/ini.v1 v1.66.4 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	github.com/valyala/fasttemplate v1.2.2 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/crypto v0.23.0 // indirect
+	golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc // indirect
+	golang.org/x/net v0.25.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/text v0.15.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	google.golang.org/protobuf v1.34.1 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )