diff --git a/build-index/cmd/cmd.go b/build-index/cmd/cmd.go index d7b0470a..b55a65ec 100644 --- a/build-index/cmd/cmd.go +++ b/build-index/cmd/cmd.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -136,7 +136,7 @@ func Run(flags *Flags, opts ...Option) { log.Fatalf("Error creating simple store: %s", err) } - backends, err := backend.NewManager(config.Backends, config.Auth, stats) + backends, err := backend.NewManager(config.BackendManager, config.Backends, config.Auth, stats) if err != nil { log.Fatalf("Error creating backend manager: %s", err) } diff --git a/build-index/cmd/config.go b/build-index/cmd/config.go index 2a206cb4..066799c2 100644 --- a/build-index/cmd/config.go +++ b/build-index/cmd/config.go @@ -34,6 +34,7 @@ import ( type Config struct { ZapLogging zap.Config `yaml:"zap"` Metrics metrics.Config `yaml:"metrics"` + BackendManager backend.ManagerConfig `yaml:"backend_manager"` Backends []backend.Config `yaml:"backends"` Auth backend.AuthConfig `yaml:"auth"` TagServer tagserver.Config `yaml:"tagserver"` diff --git a/lib/backend/client.go b/lib/backend/client.go index cad2a8f9..317e6c75 100644 --- a/lib/backend/client.go +++ b/lib/backend/client.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -17,15 +17,16 @@ import ( "fmt" "io" - "github.com/uber/kraken/core" "github.com/uber-go/tally" + "github.com/uber/kraken/core" + "go.uber.org/zap" ) var _factories = make(map[string]ClientFactory) // ClientFactory creates backend client given name. type ClientFactory interface { - Create(config interface{}, masterAuthConfig AuthConfig, stats tally.Scope) (Client, error) + Create(config interface{}, masterAuthConfig AuthConfig, stats tally.Scope, logger *zap.SugaredLogger) (Client, error) } // Register registers new Factory with corresponding backend client name. diff --git a/lib/backend/fixtures.go b/lib/backend/fixtures.go index f35b6d15..aebf64a2 100644 --- a/lib/backend/fixtures.go +++ b/lib/backend/fixtures.go @@ -13,11 +13,14 @@ // limitations under the License. package backend -import "github.com/uber-go/tally" +import ( + "github.com/uber-go/tally") + "go.uber.org/zap" +) // ManagerFixture returns a Manager with no clients for testing purposes. func ManagerFixture() *Manager { - m, err := NewManager(nil, AuthConfig{}, tally.NoopScope) + m, err := NewManager(nil, AuthConfig{}, tally.NoopScope, zap.NewNop()) if err != nil { panic(err) } diff --git a/lib/backend/gcsbackend/client.go b/lib/backend/gcsbackend/client.go index 5256faa2..4d6a026d 100644 --- a/lib/backend/gcsbackend/client.go +++ b/lib/backend/gcsbackend/client.go @@ -31,6 +31,7 @@ import ( "google.golang.org/api/iterator" "google.golang.org/api/option" "gopkg.in/yaml.v2" + "go.uber.org/zap" ) const _gcs = "gcs" @@ -42,7 +43,7 @@ func init() { type factory struct{} func (f *factory) Create( - confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) { + confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) { confBytes, err := yaml.Marshal(confRaw) if err != nil { diff --git a/lib/backend/gcsbackend/client_test.go b/lib/backend/gcsbackend/client_test.go index ae7dbe77..d543eaa3 100644 --- a/lib/backend/gcsbackend/client_test.go +++ b/lib/backend/gcsbackend/client_test.go @@ -34,6 +34,7 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + "go.uber.org/zap" ) type clientMocks struct { @@ -85,7 +86,7 @@ func TestClientFactory(t *testing.T) { userAuth := UserAuthConfig{"test-user": auth} masterAuth := backend.AuthConfig{_gcs: userAuth} f := factory{} - _, err := f.Create(config, masterAuth, tally.NoopScope) + _, err := f.Create(config, masterAuth, tally.NoopScope, zap.NewNop().Sugar()) fmt.Println(err.Error()) require.True(strings.Contains(err.Error(), "invalid gcs credentials")) } diff --git a/lib/backend/hdfsbackend/client.go b/lib/backend/hdfsbackend/client.go index 26f396f1..ca4ccb16 100644 --- a/lib/backend/hdfsbackend/client.go +++ b/lib/backend/hdfsbackend/client.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -30,6 +30,7 @@ import ( "github.com/uber/kraken/utils/log" "github.com/satori/go.uuid" + "go.uber.org/zap" "gopkg.in/yaml.v2" ) @@ -42,7 +43,7 @@ func init() { type factory struct{} func (f *factory) Create( - confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) { + confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) { confBytes, err := yaml.Marshal(confRaw) if err != nil { diff --git a/lib/backend/hdfsbackend/client_test.go b/lib/backend/hdfsbackend/client_test.go index aa4560bf..56e84dd1 100644 --- a/lib/backend/hdfsbackend/client_test.go +++ b/lib/backend/hdfsbackend/client_test.go @@ -28,6 +28,7 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + "go.uber.org/zap" ) type clientMocks struct { @@ -64,7 +65,7 @@ func TestClientFactory(t *testing.T) { testing: true, } f := factory{} - _, err := f.Create(config, nil, tally.NoopScope) + _, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar()) require.NoError(err) } diff --git a/lib/backend/httpbackend/http_test.go b/lib/backend/httpbackend/http_test.go index a38e5b6d..cb320d7d 100644 --- a/lib/backend/httpbackend/http_test.go +++ b/lib/backend/httpbackend/http_test.go @@ -28,6 +28,7 @@ import ( "github.com/go-chi/chi" "github.com/stretchr/testify/require" + "go.uber.org/zap" ) func TestClientFactory(t *testing.T) { @@ -35,7 +36,7 @@ func TestClientFactory(t *testing.T) { config := Config{} f := factory{} - _, err := f.Create(config, nil, tally.NoopScope) + _, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar()) require.NoError(err) } diff --git a/lib/backend/manager.go b/lib/backend/manager.go index 35069ab6..9ebe19e4 100644 --- a/lib/backend/manager.go +++ b/lib/backend/manager.go @@ -53,8 +53,19 @@ type Manager struct { backends []*backend } +// ManagerConfig is config for backend manager. +type ManagerConfig struct { + Log log.Config `yaml:"log"` +} + // NewManager creates a new backend Manager. -func NewManager(configs []Config, auth AuthConfig, stats tally.Scope) (*Manager, error) { +func NewManager(managerConfig ManagerConfig, configs []Config, auth AuthConfig, stats tally.Scope) (*Manager, error) { + logger, err := log.New(managerConfig.Log, nil) + if err != nil { + return nil, fmt.Errorf("log: %s", err) + } + slogger := logger.Sugar() + var backends []*backend for _, config := range configs { config = config.applyDefaults() @@ -71,7 +82,7 @@ func NewManager(configs []Config, auth AuthConfig, stats tally.Scope) (*Manager, if err != nil { return nil, fmt.Errorf("get backend client factory: %s", err) } - c, err = factory.Create(backendConfig, auth, stats) + c, err = factory.Create(backendConfig, auth, stats, slogger) if err != nil { return nil, fmt.Errorf("create backend client: %s", err) } diff --git a/lib/backend/registrybackend/blobclient_test.go b/lib/backend/registrybackend/blobclient_test.go index 68d6cc14..bd84fc30 100644 --- a/lib/backend/registrybackend/blobclient_test.go +++ b/lib/backend/registrybackend/blobclient_test.go @@ -28,6 +28,7 @@ import ( "github.com/uber/kraken/utils/memsize" "github.com/uber/kraken/utils/randutil" "github.com/uber/kraken/utils/testutil" + "go.uber.org/zap" ) func TestClientFactory(t *testing.T) { @@ -35,7 +36,7 @@ func TestClientFactory(t *testing.T) { config := Config{} f := blobClientFactory{} - _, err := f.Create(config, nil, tally.NoopScope) + _, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar()) require.NoError(err) } diff --git a/lib/backend/s3backend/client.go b/lib/backend/s3backend/client.go index d2416afe..debbcfbc 100644 --- a/lib/backend/s3backend/client.go +++ b/lib/backend/s3backend/client.go @@ -34,6 +34,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" "gopkg.in/yaml.v2" + "go.uber.org/zap" ) const _s3 = "s3" @@ -45,7 +46,7 @@ func init() { type factory struct{} func (f *factory) Create( - confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) { + confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) { confBytes, err := yaml.Marshal(confRaw) if err != nil { diff --git a/lib/backend/s3backend/client_test.go b/lib/backend/s3backend/client_test.go index 1ae69111..21cd09f9 100644 --- a/lib/backend/s3backend/client_test.go +++ b/lib/backend/s3backend/client_test.go @@ -30,6 +30,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + "go.uber.org/zap" ) type clientMocks struct { @@ -82,7 +83,7 @@ func TestClientFactory(t *testing.T) { userAuth := UserAuthConfig{"test-user": auth} masterAuth := backend.AuthConfig{_s3: userAuth} f := factory{} - _, err := f.Create(config, masterAuth, tally.NoopScope) + _, err := f.Create(config, masterAuth, tally.NoopScope, zap.NewNop().Sugar()) require.NoError(err) } diff --git a/lib/backend/shadowbackend/client.go b/lib/backend/shadowbackend/client.go index 7e8cbb5a..6fd962c9 100644 --- a/lib/backend/shadowbackend/client.go +++ b/lib/backend/shadowbackend/client.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -27,6 +27,7 @@ import ( "github.com/uber/kraken/lib/backend/sqlbackend" "github.com/uber/kraken/lib/backend/testfs" "github.com/uber/kraken/utils/log" + "go.uber.org/zap" "gopkg.in/yaml.v2" ) @@ -43,7 +44,7 @@ func (f *factory) Name() string { } func (f *factory) Create( - confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) { + confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) { confBytes, err := yaml.Marshal(confRaw) if err != nil { diff --git a/lib/backend/shadowbackend/client_test.go b/lib/backend/shadowbackend/client_test.go index 14fdac99..bdd5d2fc 100644 --- a/lib/backend/shadowbackend/client_test.go +++ b/lib/backend/shadowbackend/client_test.go @@ -34,6 +34,7 @@ import ( "github.com/uber/kraken/lib/backend/s3backend" "github.com/uber/kraken/lib/backend/sqlbackend" "github.com/uber/kraken/lib/backend/testfs" + "go.uber.org/zap" ) type clientMocks struct { @@ -135,7 +136,7 @@ func TestClientFactory(t *testing.T) { }, } { f := factory{} - c, err := f.Create(tc.config, tc.masterAuthConfig, tally.NoopScope) + c, err := f.Create(tc.config, tc.masterAuthConfig, tally.NoopScope, zap.NewNop().Sugar()) if tc.wantErrMsg == "" { require.NoError(t, err) require.NotNil(t, c) diff --git a/lib/backend/sqlbackend/client.go b/lib/backend/sqlbackend/client.go index aadf0346..4ecbf173 100644 --- a/lib/backend/sqlbackend/client.go +++ b/lib/backend/sqlbackend/client.go @@ -28,6 +28,7 @@ import ( "github.com/uber/kraken/core" "github.com/uber/kraken/lib/backend" "github.com/uber/kraken/lib/backend/backenderrors" + "go.uber.org/zap" "gopkg.in/yaml.v2" ) @@ -44,7 +45,7 @@ func (f *factory) Name() string { } func (f *factory) Create( - confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope) (backend.Client, error) { + confRaw interface{}, masterAuthConfig backend.AuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (backend.Client, error) { confBytes, err := yaml.Marshal(confRaw) if err != nil { @@ -76,7 +77,7 @@ type Client struct { } // NewClient creates a new Client for a SQL database. -func NewClient(config Config, authConfig UserAuthConfig, stats tally.Scope) (*Client, error) { +func NewClient(config Config, authConfig UserAuthConfig, stats tally.Scope, _ *zap.SugaredLogger) (*Client, error) { conStr, err := getDBConnectionString(config, authConfig) if err != nil { return nil, fmt.Errorf("error building database connection string: %v", err) diff --git a/lib/backend/sqlbackend/client_test.go b/lib/backend/sqlbackend/client_test.go index 8f5cd2d4..6e49e839 100644 --- a/lib/backend/sqlbackend/client_test.go +++ b/lib/backend/sqlbackend/client_test.go @@ -24,6 +24,7 @@ import ( "github.com/uber-go/tally" "github.com/uber/kraken/core" "github.com/uber/kraken/lib/backend/backenderrors" + "go.uber.org/zap" ) func generateSingleTag(sqlClient *Client, repo string, tag string) Tag { @@ -48,7 +49,7 @@ func TestClientFactory(t *testing.T) { config := Config{Dialect: "sqlite3", ConnectionString: ":memory:"} f := factory{} - _, err := f.Create(config, nil, tally.NoopScope) + _, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar()) require.NoError(t, err) } @@ -56,7 +57,7 @@ func TestClientFactoryAuth(t *testing.T) { config := Config{Dialect: "sqlite3", ConnectionString: ":memory:"} f := factory{} - _, err := f.Create(config, nil, tally.NoopScope) + _, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar()) require.NoError(t, err) } diff --git a/lib/backend/testfs/client_test.go b/lib/backend/testfs/client_test.go index 0f6f935f..e9d72fdd 100644 --- a/lib/backend/testfs/client_test.go +++ b/lib/backend/testfs/client_test.go @@ -20,6 +20,7 @@ import ( "github.com/uber/kraken/lib/backend/namepath" "github.com/stretchr/testify/require" + "go.uber.org/zap" ) func TestClientFactory(t *testing.T) { @@ -30,6 +31,6 @@ func TestClientFactory(t *testing.T) { NamePath: namepath.Identity, } f := factory{} - _, err := f.Create(config, nil, tally.NoopScope) + _, err := f.Create(config, nil, tally.NoopScope, zap.NewNop().Sugar()) require.NoError(err) } diff --git a/origin/cmd/cmd.go b/origin/cmd/cmd.go index 42e344d8..cb0c3f80 100644 --- a/origin/cmd/cmd.go +++ b/origin/cmd/cmd.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -186,7 +186,7 @@ func Run(flags *Flags, opts ...Option) { log.Fatalf("Failed to create peer context: %s", err) } - backendManager, err := backend.NewManager(config.Backends, config.Auth, stats) + backendManager, err := backend.NewManager(config.BackendManager, config.Backends, config.Auth, stats) if err != nil { log.Fatalf("Error creating backend manager: %s", err) } diff --git a/origin/cmd/config.go b/origin/cmd/config.go index e191d27a..bf0b651f 100644 --- a/origin/cmd/config.go +++ b/origin/cmd/config.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -37,23 +37,24 @@ import ( // Config defines origin server configuration. // TODO(evelynl94): consolidate cluster and hashring. type Config struct { - Verbose bool - ZapLogging zap.Config `yaml:"zap"` - Cluster hostlist.Config `yaml:"cluster"` - HashRing hashring.Config `yaml:"hashring"` - HealthCheck healthcheck.FilterConfig `yaml:"healthcheck"` - BlobServer blobserver.Config `yaml:"blobserver"` - CAStore store.CAStoreConfig `yaml:"castore"` - Scheduler scheduler.Config `yaml:"scheduler"` - NetworkEvent networkevent.Config `yaml:"network_event"` - PeerIDFactory core.PeerIDFactory `yaml:"peer_id_factory"` - Metrics metrics.Config `yaml:"metrics"` - MetaInfoGen metainfogen.Config `yaml:"metainfogen"` - Backends []backend.Config `yaml:"backends"` - Auth backend.AuthConfig `yaml:"auth"` - BlobRefresh blobrefresh.Config `yaml:"blobrefresh"` - LocalDB localdb.Config `yaml:"localdb"` - WriteBack persistedretry.Config `yaml:"writeback"` - Nginx nginx.Config `yaml:"nginx"` - TLS httputil.TLSConfig `yaml:"tls"` + Verbose bool + ZapLogging zap.Config `yaml:"zap"` + Cluster hostlist.Config `yaml:"cluster"` + HashRing hashring.Config `yaml:"hashring"` + HealthCheck healthcheck.FilterConfig `yaml:"healthcheck"` + BlobServer blobserver.Config `yaml:"blobserver"` + CAStore store.CAStoreConfig `yaml:"castore"` + Scheduler scheduler.Config `yaml:"scheduler"` + NetworkEvent networkevent.Config `yaml:"network_event"` + PeerIDFactory core.PeerIDFactory `yaml:"peer_id_factory"` + Metrics metrics.Config `yaml:"metrics"` + MetaInfoGen metainfogen.Config `yaml:"metainfogen"` + BackendManager backend.ManagerConfig `yaml:"backend_manager"` + Backends []backend.Config `yaml:"backends"` + Auth backend.AuthConfig `yaml:"auth"` + BlobRefresh blobrefresh.Config `yaml:"blobrefresh"` + LocalDB localdb.Config `yaml:"localdb"` + WriteBack persistedretry.Config `yaml:"writeback"` + Nginx nginx.Config `yaml:"nginx"` + TLS httputil.TLSConfig `yaml:"tls"` }