logs for backend manager and backends

uber · Jan 17, 2025 · d465f85 · d465f85
1 parent e757d98
commit d465f85
Show file tree

Hide file tree

Showing 19 changed files with 73 additions and 46 deletions.
diff --git a/build-index/cmd/cmd.go b/build-index/cmd/cmd.go
@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -136,7 +136,7 @@ func Run(flags *Flags, opts ...Option) {
 		log.Fatalf("Error creating simple store: %s", err)
 	}
 
-	backends, err := backend.NewManager(config.Backends, config.Auth, stats)
+	backends, err := backend.NewManager(config.BackendManager, config.Backends, config.Auth, stats)
 	if err != nil {
 		log.Fatalf("Error creating backend manager: %s", err)
 	}

diff --git a/build-index/cmd/config.go b/build-index/cmd/config.go
@@ -34,6 +34,7 @@ import (
 type Config struct {
 	ZapLogging     zap.Config                   `yaml:"zap"`
 	Metrics        metrics.Config               `yaml:"metrics"`
+  BackendManager backend.ManagerConfig   `yaml:"backend_manager"`
 	Backends       []backend.Config             `yaml:"backends"`
 	Auth           backend.AuthConfig           `yaml:"auth"`
 	TagServer      tagserver.Config             `yaml:"tagserver"`

diff --git a/lib/backend/client.go b/lib/backend/client.go
@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,15 +17,16 @@ import (
 	"fmt"
 	"io"
 
-	"github.com/uber/kraken/core"
 	"github.com/uber-go/tally"
+	"github.com/uber/kraken/core"
+	"go.uber.org/zap"
 )
 
 var _factories = make(map[string]ClientFactory)
 
 // ClientFactory creates backend client given name.
 type ClientFactory interface {
-	Create(config interface{}, masterAuthConfig AuthConfig, stats tally.Scope) (Client, error)
+	Create(config interface{}, masterAuthConfig AuthConfig, stats tally.Scope, logger *zap.SugaredLogger) (Client, error)
 }
 
 // Register registers new Factory with corresponding backend client name.

diff --git a/lib/backend/gcsbackend/client.go b/lib/backend/gcsbackend/client.go
@@ -31,6 +31,7 @@ import (
 	"google.golang.org/api/iterator"
 	"google.golang.org/api/option"
 	"gopkg.in/yaml.v2"
+  "go.uber.org/zap"
 )
 
 const _gcs = "gcs"
@@ -42,7 +43,7 @@ func init() {
 type factory struct{}
 
 func (f *factory) Create(
-	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) {
+	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) {
 
 	confBytes, err := yaml.Marshal(confRaw)
 	if err != nil {

diff --git a/lib/backend/gcsbackend/client_test.go b/lib/backend/gcsbackend/client_test.go
@@ -34,6 +34,7 @@ import (
 
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
 )
 
 type clientMocks struct {
@@ -85,7 +86,7 @@ func TestClientFactory(t *testing.T) {
 	userAuth := UserAuthConfig{"test-user": auth}
 	masterAuth := backend.AuthConfig{_gcs: userAuth}
 	f := factory{}
-	_, err := f.Create(config, masterAuth, tally.NoopScope)
+	_, err := f.Create(config, masterAuth, tally.NoopScope, zap.NewNop().Sugar())
 	fmt.Println(err.Error())
 	require.True(strings.Contains(err.Error(), "invalid gcs credentials"))
 }

diff --git a/lib/backend/hdfsbackend/client.go b/lib/backend/hdfsbackend/client.go
@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -30,6 +30,7 @@ import (
 	"github.com/uber/kraken/utils/log"
 
 	"github.com/satori/go.uuid"
+	"go.uber.org/zap"
 	"gopkg.in/yaml.v2"
 )
 
@@ -42,7 +43,7 @@ func init() {
 type factory struct{}
 
 func (f *factory) Create(
-	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) {
+	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) {
 
 	confBytes, err := yaml.Marshal(confRaw)
 	if err != nil {

diff --git a/lib/backend/hdfsbackend/client_test.go b/lib/backend/hdfsbackend/client_test.go
@@ -28,6 +28,7 @@ import (
 
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
 )
 
 type clientMocks struct {
@@ -64,7 +65,7 @@ func TestClientFactory(t *testing.T) {
 		testing:       true,
 	}
 	f := factory{}
-	_, err := f.Create(config, nil, tally.NoopScope)
+	_, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar())
 	require.NoError(err)
 }
 

diff --git a/lib/backend/httpbackend/http_test.go b/lib/backend/httpbackend/http_test.go
@@ -28,14 +28,15 @@ import (
 
 	"github.com/go-chi/chi"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
 )
 
 func TestClientFactory(t *testing.T) {
 	require := require.New(t)
 
 	config := Config{}
 	f := factory{}
-	_, err := f.Create(config, nil, tally.NoopScope)
+	_, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar())
 	require.NoError(err)
 }
 

diff --git a/lib/backend/manager.go b/lib/backend/manager.go
@@ -53,8 +53,19 @@ type Manager struct {
 	backends []*backend
 }
 
+// ManagerConfig is config for backend manager.
+type ManagerConfig struct {
+	Log log.Config `yaml:"log"`
+}
+
 // NewManager creates a new backend Manager.
-func NewManager(configs []Config, auth AuthConfig, stats tally.Scope) (*Manager, error) {
+func NewManager(managerConfig ManagerConfig, configs []Config, auth AuthConfig, stats tally.Scope) (*Manager, error) {
+	logger, err := log.New(managerConfig.Log, nil)
+	if err != nil {
+		return nil, fmt.Errorf("log: %s", err)
+	}
+	slogger := logger.Sugar()
+
 	var backends []*backend
 	for _, config := range configs {
 		config = config.applyDefaults()
@@ -71,7 +82,7 @@ func NewManager(configs []Config, auth AuthConfig, stats tally.Scope) (*Manager,
 		if err != nil {
 			return nil, fmt.Errorf("get backend client factory: %s", err)
 		}
-		c, err = factory.Create(backendConfig, auth, stats)
+		c, err = factory.Create(backendConfig, auth, stats, slogger)
 		if err != nil {
 			return nil, fmt.Errorf("create backend client: %s", err)
 		}

diff --git a/lib/backend/registrybackend/blobclient_test.go b/lib/backend/registrybackend/blobclient_test.go
@@ -28,14 +28,15 @@ import (
 	"github.com/uber/kraken/utils/memsize"
 	"github.com/uber/kraken/utils/randutil"
 	"github.com/uber/kraken/utils/testutil"
+	"go.uber.org/zap"
 )
 
 func TestClientFactory(t *testing.T) {
 	require := require.New(t)
 
 	config := Config{}
 	f := blobClientFactory{}
-	_, err := f.Create(config, nil, tally.NoopScope)
+	_, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar())
 	require.NoError(err)
 }
 

diff --git a/lib/backend/s3backend/client.go b/lib/backend/s3backend/client.go
@@ -34,6 +34,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"gopkg.in/yaml.v2"
+	"go.uber.org/zap"
 )
 
 const _s3 = "s3"
@@ -45,7 +46,7 @@ func init() {
 type factory struct{}
 
 func (f *factory) Create(
-	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) {
+	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) {
 
 	confBytes, err := yaml.Marshal(confRaw)
 	if err != nil {

diff --git a/lib/backend/s3backend/client_test.go b/lib/backend/s3backend/client_test.go
@@ -30,6 +30,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
 )
 
 type clientMocks struct {
@@ -82,7 +83,7 @@ func TestClientFactory(t *testing.T) {
 	userAuth := UserAuthConfig{"test-user": auth}
 	masterAuth := backend.AuthConfig{_s3: userAuth}
 	f := factory{}
-	_, err := f.Create(config, masterAuth, tally.NoopScope)
+	_, err := f.Create(config, masterAuth, tally.NoopScope, zap.NewNop().Sugar())
 	require.NoError(err)
 }
 

diff --git a/lib/backend/shadowbackend/client.go b/lib/backend/shadowbackend/client.go
@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -27,6 +27,7 @@ import (
 	"github.com/uber/kraken/lib/backend/sqlbackend"
 	"github.com/uber/kraken/lib/backend/testfs"
 	"github.com/uber/kraken/utils/log"
+	"go.uber.org/zap"
 	"gopkg.in/yaml.v2"
 )
 
@@ -43,7 +44,7 @@ func (f *factory) Name() string {
 }
 
 func (f *factory) Create(
-	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) {
+	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) {
 
 	confBytes, err := yaml.Marshal(confRaw)
 	if err != nil {

diff --git a/lib/backend/shadowbackend/client_test.go b/lib/backend/shadowbackend/client_test.go
@@ -34,6 +34,7 @@ import (
 	"github.com/uber/kraken/lib/backend/s3backend"
 	"github.com/uber/kraken/lib/backend/sqlbackend"
 	"github.com/uber/kraken/lib/backend/testfs"
+	"go.uber.org/zap"
 )
 
 type clientMocks struct {
@@ -135,7 +136,7 @@ func TestClientFactory(t *testing.T) {
 		},
 	} {
 		f := factory{}
-		c, err := f.Create(tc.config, tc.masterAuthConfig, tally.NoopScope)
+		c, err := f.Create(tc.config, tc.masterAuthConfig, tally.NoopScope, zap.NewNop().Sugar())
 		if tc.wantErrMsg == "" {
 			require.NoError(t, err)
 			require.NotNil(t, c)

diff --git a/lib/backend/sqlbackend/client.go b/lib/backend/sqlbackend/client.go
@@ -28,6 +28,7 @@ import (
 	"github.com/uber/kraken/core"
 	"github.com/uber/kraken/lib/backend"
 	"github.com/uber/kraken/lib/backend/backenderrors"
+	"go.uber.org/zap"
 	"gopkg.in/yaml.v2"
 )
 
@@ -44,7 +45,7 @@ func (f *factory) Name() string {
 }
 
 func (f *factory) Create(
-	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) {
+	confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) {
 
 	confBytes, err := yaml.Marshal(confRaw)
 	if err != nil {
@@ -76,7 +77,7 @@ type Client struct {
 }
 
 // NewClient creates a new Client for a SQL database.
-func NewClient(config Config, authConfig UserAuthConfig, stats tally.Scope) (*Client, error) {
+func NewClient(config Config, authConfig UserAuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (*Client, error) {
 	conStr, err := getDBConnectionString(config, authConfig)
 	if err != nil {
 		return nil, fmt.Errorf("error building database connection string: %v", err)

diff --git a/lib/backend/sqlbackend/client_test.go b/lib/backend/sqlbackend/client_test.go
@@ -24,6 +24,7 @@ import (
 	"github.com/uber-go/tally"
 	"github.com/uber/kraken/core"
 	"github.com/uber/kraken/lib/backend/backenderrors"
+	"go.uber.org/zap"
 )
 
 func generateSingleTag(sqlClient *Client, repo string, tag string) Tag {
@@ -48,15 +49,15 @@ func TestClientFactory(t *testing.T) {
 	config := Config{Dialect: "sqlite3", ConnectionString: ":memory:"}
 
 	f := factory{}
-	_, err := f.Create(config, nil, tally.NoopScope)
+	_, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar())
 	require.NoError(t, err)
 }
 
 func TestClientFactoryAuth(t *testing.T) {
 	config := Config{Dialect: "sqlite3", ConnectionString: ":memory:"}
 
 	f := factory{}
-	_, err := f.Create(config, nil, tally.NoopScope)
+	_, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar())
 	require.NoError(t, err)
 }
 

diff --git a/lib/backend/testfs/client_test.go b/lib/backend/testfs/client_test.go
@@ -20,6 +20,7 @@ import (
 	"github.com/uber/kraken/lib/backend/namepath"
 
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
 )
 
 func TestClientFactory(t *testing.T) {
@@ -30,6 +31,6 @@ func TestClientFactory(t *testing.T) {
 		NamePath: namepath.Identity,
 	}
 	f := factory{}
-	_, err := f.Create(config, nil, tally.NoopScope)
+	_, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar())
 	require.NoError(err)
 }
diff --git a/origin/cmd/cmd.go b/origin/cmd/cmd.go
@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -186,7 +186,7 @@ func Run(flags *Flags, opts ...Option) {
 		log.Fatalf("Failed to create peer context: %s", err)
 	}
 
-	backendManager, err := backend.NewManager(config.Backends, config.Auth, stats)
+	backendManager, err := backend.NewManager(config.BackendManager, config.Backends, config.Auth, stats)
 	if err != nil {
 		log.Fatalf("Error creating backend manager: %s", err)
 	}