From 239e5e23f9e70edea128103577b20f8fcfbf5594 Mon Sep 17 00:00:00 2001 From: "zach.cheung" Date: Thu, 16 Jan 2025 23:10:14 +0000 Subject: [PATCH] logs for backend manager and backends --- build-index/cmd/cmd.go | 4 +- build-index/cmd/config.go | 1 + lib/backend/client.go | 5 ++- lib/backend/gcsbackend/client.go | 3 +- lib/backend/gcsbackend/client_test.go | 3 +- lib/backend/hdfsbackend/client.go | 3 +- lib/backend/hdfsbackend/client_test.go | 3 +- lib/backend/httpbackend/http_test.go | 3 +- lib/backend/manager.go | 15 ++++++- .../registrybackend/blobclient_test.go | 3 +- lib/backend/s3backend/client.go | 3 +- lib/backend/s3backend/client_test.go | 3 +- lib/backend/shadowbackend/client.go | 5 ++- lib/backend/shadowbackend/client_test.go | 3 +- lib/backend/sqlbackend/client.go | 5 ++- lib/backend/sqlbackend/client_test.go | 5 ++- lib/backend/testfs/client_test.go | 3 +- origin/cmd/cmd.go | 4 +- origin/cmd/config.go | 41 ++++++++++--------- 19 files changed, 71 insertions(+), 44 deletions(-) diff --git a/build-index/cmd/cmd.go b/build-index/cmd/cmd.go index d7b0470a9..b55a65ec3 100644 --- a/build-index/cmd/cmd.go +++ b/build-index/cmd/cmd.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -136,7 +136,7 @@ func Run(flags *Flags, opts ...Option) { log.Fatalf("Error creating simple store: %s", err) } - backends, err := backend.NewManager(config.Backends, config.Auth, stats) + backends, err := backend.NewManager(config.BackendManager, config.Backends, config.Auth, stats) if err != nil { log.Fatalf("Error creating backend manager: %s", err) } diff --git a/build-index/cmd/config.go b/build-index/cmd/config.go index 2a206cb40..066799c2d 100644 --- a/build-index/cmd/config.go +++ b/build-index/cmd/config.go @@ -34,6 +34,7 @@ import ( type Config struct { ZapLogging zap.Config `yaml:"zap"` Metrics metrics.Config `yaml:"metrics"` + BackendManager backend.ManagerConfig `yaml:"backend_manager"` Backends []backend.Config `yaml:"backends"` Auth backend.AuthConfig `yaml:"auth"` TagServer tagserver.Config `yaml:"tagserver"` diff --git a/lib/backend/client.go b/lib/backend/client.go index cad2a8f99..bf50172a1 100644 --- a/lib/backend/client.go +++ b/lib/backend/client.go @@ -19,13 +19,14 @@ import ( "github.com/uber/kraken/core" "github.com/uber-go/tally" + "go.uber.org/zap" ) var _factories = make(map[string]ClientFactory) // ClientFactory creates backend client given name. -type ClientFactory interface { - Create(config interface{}, masterAuthConfig AuthConfig, stats tally.Scope) (Client, error) +type ClientFactory interface{ + Create(config interface{}, masterAuthConfig AuthConfig, stats tally.Scope, logger *zap.SugaredLogger) (Client, error) } // Register registers new Factory with corresponding backend client name. diff --git a/lib/backend/gcsbackend/client.go b/lib/backend/gcsbackend/client.go index 5256faa2c..4d6a026d2 100644 --- a/lib/backend/gcsbackend/client.go +++ b/lib/backend/gcsbackend/client.go @@ -31,6 +31,7 @@ import ( "google.golang.org/api/iterator" "google.golang.org/api/option" "gopkg.in/yaml.v2" + "go.uber.org/zap" ) const _gcs = "gcs" @@ -42,7 +43,7 @@ func init() { type factory struct{} func (f *factory) Create( - confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) { + confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) { confBytes, err := yaml.Marshal(confRaw) if err != nil { diff --git a/lib/backend/gcsbackend/client_test.go b/lib/backend/gcsbackend/client_test.go index ae7dbe771..d543eaa34 100644 --- a/lib/backend/gcsbackend/client_test.go +++ b/lib/backend/gcsbackend/client_test.go @@ -34,6 +34,7 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + "go.uber.org/zap" ) type clientMocks struct { @@ -85,7 +86,7 @@ func TestClientFactory(t *testing.T) { userAuth := UserAuthConfig{"test-user": auth} masterAuth := backend.AuthConfig{_gcs: userAuth} f := factory{} - _, err := f.Create(config, masterAuth, tally.NoopScope) + _, err := f.Create(config, masterAuth, tally.NoopScope, zap.NewNop().Sugar()) fmt.Println(err.Error()) require.True(strings.Contains(err.Error(), "invalid gcs credentials")) } diff --git a/lib/backend/hdfsbackend/client.go b/lib/backend/hdfsbackend/client.go index 26f396f13..bf6e16cda 100644 --- a/lib/backend/hdfsbackend/client.go +++ b/lib/backend/hdfsbackend/client.go @@ -31,6 +31,7 @@ import ( "github.com/satori/go.uuid" "gopkg.in/yaml.v2" + "go.uber.org/zap" ) const _hdfs = "hdfs" @@ -42,7 +43,7 @@ func init() { type factory struct{} func (f *factory) Create( - confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) { + confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) { confBytes, err := yaml.Marshal(confRaw) if err != nil { diff --git a/lib/backend/hdfsbackend/client_test.go b/lib/backend/hdfsbackend/client_test.go index aa4560bfe..56e84dd17 100644 --- a/lib/backend/hdfsbackend/client_test.go +++ b/lib/backend/hdfsbackend/client_test.go @@ -28,6 +28,7 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + "go.uber.org/zap" ) type clientMocks struct { @@ -64,7 +65,7 @@ func TestClientFactory(t *testing.T) { testing: true, } f := factory{} - _, err := f.Create(config, nil, tally.NoopScope) + _, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar()) require.NoError(err) } diff --git a/lib/backend/httpbackend/http_test.go b/lib/backend/httpbackend/http_test.go index a38e5b6d0..cb320d7da 100644 --- a/lib/backend/httpbackend/http_test.go +++ b/lib/backend/httpbackend/http_test.go @@ -28,6 +28,7 @@ import ( "github.com/go-chi/chi" "github.com/stretchr/testify/require" + "go.uber.org/zap" ) func TestClientFactory(t *testing.T) { @@ -35,7 +36,7 @@ func TestClientFactory(t *testing.T) { config := Config{} f := factory{} - _, err := f.Create(config, nil, tally.NoopScope) + _, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar()) require.NoError(err) } diff --git a/lib/backend/manager.go b/lib/backend/manager.go index 35069ab65..9ebe19e46 100644 --- a/lib/backend/manager.go +++ b/lib/backend/manager.go @@ -53,8 +53,19 @@ type Manager struct { backends []*backend } +// ManagerConfig is config for backend manager. +type ManagerConfig struct { + Log log.Config `yaml:"log"` +} + // NewManager creates a new backend Manager. -func NewManager(configs []Config, auth AuthConfig, stats tally.Scope) (*Manager, error) { +func NewManager(managerConfig ManagerConfig, configs []Config, auth AuthConfig, stats tally.Scope) (*Manager, error) { + logger, err := log.New(managerConfig.Log, nil) + if err != nil { + return nil, fmt.Errorf("log: %s", err) + } + slogger := logger.Sugar() + var backends []*backend for _, config := range configs { config = config.applyDefaults() @@ -71,7 +82,7 @@ func NewManager(configs []Config, auth AuthConfig, stats tally.Scope) (*Manager, if err != nil { return nil, fmt.Errorf("get backend client factory: %s", err) } - c, err = factory.Create(backendConfig, auth, stats) + c, err = factory.Create(backendConfig, auth, stats, slogger) if err != nil { return nil, fmt.Errorf("create backend client: %s", err) } diff --git a/lib/backend/registrybackend/blobclient_test.go b/lib/backend/registrybackend/blobclient_test.go index 68d6cc149..bd84fc30f 100644 --- a/lib/backend/registrybackend/blobclient_test.go +++ b/lib/backend/registrybackend/blobclient_test.go @@ -28,6 +28,7 @@ import ( "github.com/uber/kraken/utils/memsize" "github.com/uber/kraken/utils/randutil" "github.com/uber/kraken/utils/testutil" + "go.uber.org/zap" ) func TestClientFactory(t *testing.T) { @@ -35,7 +36,7 @@ func TestClientFactory(t *testing.T) { config := Config{} f := blobClientFactory{} - _, err := f.Create(config, nil, tally.NoopScope) + _, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar()) require.NoError(err) } diff --git a/lib/backend/s3backend/client.go b/lib/backend/s3backend/client.go index d2416afe1..debbcfbcd 100644 --- a/lib/backend/s3backend/client.go +++ b/lib/backend/s3backend/client.go @@ -34,6 +34,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" "gopkg.in/yaml.v2" + "go.uber.org/zap" ) const _s3 = "s3" @@ -45,7 +46,7 @@ func init() { type factory struct{} func (f *factory) Create( - confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) { + confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) { confBytes, err := yaml.Marshal(confRaw) if err != nil { diff --git a/lib/backend/s3backend/client_test.go b/lib/backend/s3backend/client_test.go index 1ae691116..21cd09f94 100644 --- a/lib/backend/s3backend/client_test.go +++ b/lib/backend/s3backend/client_test.go @@ -30,6 +30,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + "go.uber.org/zap" ) type clientMocks struct { @@ -82,7 +83,7 @@ func TestClientFactory(t *testing.T) { userAuth := UserAuthConfig{"test-user": auth} masterAuth := backend.AuthConfig{_s3: userAuth} f := factory{} - _, err := f.Create(config, masterAuth, tally.NoopScope) + _, err := f.Create(config, masterAuth, tally.NoopScope, zap.NewNop().Sugar()) require.NoError(err) } diff --git a/lib/backend/shadowbackend/client.go b/lib/backend/shadowbackend/client.go index 7e8cbb5a7..6fd962c9d 100644 --- a/lib/backend/shadowbackend/client.go +++ b/lib/backend/shadowbackend/client.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -27,6 +27,7 @@ import ( "github.com/uber/kraken/lib/backend/sqlbackend" "github.com/uber/kraken/lib/backend/testfs" "github.com/uber/kraken/utils/log" + "go.uber.org/zap" "gopkg.in/yaml.v2" ) @@ -43,7 +44,7 @@ func (f *factory) Name() string { } func (f *factory) Create( - confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) { + confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) { confBytes, err := yaml.Marshal(confRaw) if err != nil { diff --git a/lib/backend/shadowbackend/client_test.go b/lib/backend/shadowbackend/client_test.go index 14fdac993..bdd5d2fcf 100644 --- a/lib/backend/shadowbackend/client_test.go +++ b/lib/backend/shadowbackend/client_test.go @@ -34,6 +34,7 @@ import ( "github.com/uber/kraken/lib/backend/s3backend" "github.com/uber/kraken/lib/backend/sqlbackend" "github.com/uber/kraken/lib/backend/testfs" + "go.uber.org/zap" ) type clientMocks struct { @@ -135,7 +136,7 @@ func TestClientFactory(t *testing.T) { }, } { f := factory{} - c, err := f.Create(tc.config, tc.masterAuthConfig, tally.NoopScope) + c, err := f.Create(tc.config, tc.masterAuthConfig, tally.NoopScope, zap.NewNop().Sugar()) if tc.wantErrMsg == "" { require.NoError(t, err) require.NotNil(t, c) diff --git a/lib/backend/sqlbackend/client.go b/lib/backend/sqlbackend/client.go index aadf03460..4ecbf1731 100644 --- a/lib/backend/sqlbackend/client.go +++ b/lib/backend/sqlbackend/client.go @@ -28,6 +28,7 @@ import ( "github.com/uber/kraken/core" "github.com/uber/kraken/lib/backend" "github.com/uber/kraken/lib/backend/backenderrors" + "go.uber.org/zap" "gopkg.in/yaml.v2" ) @@ -44,7 +45,7 @@ func (f *factory) Name() string { } func (f *factory) Create( - confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) { + confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) { confBytes, err := yaml.Marshal(confRaw) if err != nil { @@ -76,7 +77,7 @@ type Client struct { } // NewClient creates a new Client for a SQL database. -func NewClient(config Config, authConfig UserAuthConfig, stats tally.Scope) (*Client, error) { +func NewClient(config Config, authConfig UserAuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (*Client, error) { conStr, err := getDBConnectionString(config, authConfig) if err != nil { return nil, fmt.Errorf("error building database connection string: %v", err) diff --git a/lib/backend/sqlbackend/client_test.go b/lib/backend/sqlbackend/client_test.go index 8f5cd2d4c..6e49e839d 100644 --- a/lib/backend/sqlbackend/client_test.go +++ b/lib/backend/sqlbackend/client_test.go @@ -24,6 +24,7 @@ import ( "github.com/uber-go/tally" "github.com/uber/kraken/core" "github.com/uber/kraken/lib/backend/backenderrors" + "go.uber.org/zap" ) func generateSingleTag(sqlClient *Client, repo string, tag string) Tag { @@ -48,7 +49,7 @@ func TestClientFactory(t *testing.T) { config := Config{Dialect: "sqlite3", ConnectionString: ":memory:"} f := factory{} - _, err := f.Create(config, nil, tally.NoopScope) + _, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar()) require.NoError(t, err) } @@ -56,7 +57,7 @@ func TestClientFactoryAuth(t *testing.T) { config := Config{Dialect: "sqlite3", ConnectionString: ":memory:"} f := factory{} - _, err := f.Create(config, nil, tally.NoopScope) + _, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar()) require.NoError(t, err) } diff --git a/lib/backend/testfs/client_test.go b/lib/backend/testfs/client_test.go index 0f6f935f9..e9d72fdd3 100644 --- a/lib/backend/testfs/client_test.go +++ b/lib/backend/testfs/client_test.go @@ -20,6 +20,7 @@ import ( "github.com/uber/kraken/lib/backend/namepath" "github.com/stretchr/testify/require" + "go.uber.org/zap" ) func TestClientFactory(t *testing.T) { @@ -30,6 +31,6 @@ func TestClientFactory(t *testing.T) { NamePath: namepath.Identity, } f := factory{} - _, err := f.Create(config, nil, tally.NoopScope) + _, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar()) require.NoError(err) } diff --git a/origin/cmd/cmd.go b/origin/cmd/cmd.go index 42e344d80..cb0c3f80e 100644 --- a/origin/cmd/cmd.go +++ b/origin/cmd/cmd.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -186,7 +186,7 @@ func Run(flags *Flags, opts ...Option) { log.Fatalf("Failed to create peer context: %s", err) } - backendManager, err := backend.NewManager(config.Backends, config.Auth, stats) + backendManager, err := backend.NewManager(config.BackendManager, config.Backends, config.Auth, stats) if err != nil { log.Fatalf("Error creating backend manager: %s", err) } diff --git a/origin/cmd/config.go b/origin/cmd/config.go index e191d27a6..bf0b651f6 100644 --- a/origin/cmd/config.go +++ b/origin/cmd/config.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -37,23 +37,24 @@ import ( // Config defines origin server configuration. // TODO(evelynl94): consolidate cluster and hashring. type Config struct { - Verbose bool - ZapLogging zap.Config `yaml:"zap"` - Cluster hostlist.Config `yaml:"cluster"` - HashRing hashring.Config `yaml:"hashring"` - HealthCheck healthcheck.FilterConfig `yaml:"healthcheck"` - BlobServer blobserver.Config `yaml:"blobserver"` - CAStore store.CAStoreConfig `yaml:"castore"` - Scheduler scheduler.Config `yaml:"scheduler"` - NetworkEvent networkevent.Config `yaml:"network_event"` - PeerIDFactory core.PeerIDFactory `yaml:"peer_id_factory"` - Metrics metrics.Config `yaml:"metrics"` - MetaInfoGen metainfogen.Config `yaml:"metainfogen"` - Backends []backend.Config `yaml:"backends"` - Auth backend.AuthConfig `yaml:"auth"` - BlobRefresh blobrefresh.Config `yaml:"blobrefresh"` - LocalDB localdb.Config `yaml:"localdb"` - WriteBack persistedretry.Config `yaml:"writeback"` - Nginx nginx.Config `yaml:"nginx"` - TLS httputil.TLSConfig `yaml:"tls"` + Verbose bool + ZapLogging zap.Config `yaml:"zap"` + Cluster hostlist.Config `yaml:"cluster"` + HashRing hashring.Config `yaml:"hashring"` + HealthCheck healthcheck.FilterConfig `yaml:"healthcheck"` + BlobServer blobserver.Config `yaml:"blobserver"` + CAStore store.CAStoreConfig `yaml:"castore"` + Scheduler scheduler.Config `yaml:"scheduler"` + NetworkEvent networkevent.Config `yaml:"network_event"` + PeerIDFactory core.PeerIDFactory `yaml:"peer_id_factory"` + Metrics metrics.Config `yaml:"metrics"` + MetaInfoGen metainfogen.Config `yaml:"metainfogen"` + BackendManager backend.ManagerConfig `yaml:"backend_manager"` + Backends []backend.Config `yaml:"backends"` + Auth backend.AuthConfig `yaml:"auth"` + BlobRefresh blobrefresh.Config `yaml:"blobrefresh"` + LocalDB localdb.Config `yaml:"localdb"` + WriteBack persistedretry.Config `yaml:"writeback"` + Nginx nginx.Config `yaml:"nginx"` + TLS httputil.TLSConfig `yaml:"tls"` }